{
    "base_speed_report": {
        "cuda_eval_elapsed_time": 44.131452,
        "eval_elapsed_time": 48.512564
    },
    "checkpoints": {
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--1d4e4bc55a9a2bf2/checkpoint-135000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 70.49595141700405,
                "HasAns_f1": 77.05313447818509,
                "HasAns_total": 5928,
                "NoAns_exact": 74.01177460050462,
                "NoAns_f1": 74.01177460050462,
                "NoAns_total": 5945,
                "best_exact": 72.25638002189842,
                "best_exact_thresh": 0.0,
                "best_f1": 75.53027719924893,
                "best_f1_thresh": 0.0,
                "exact": 72.25638002189842,
                "f1": 75.53027719924886,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 5.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.228707473754884,
                "eval_elapsed_time": 30.511660314979963
            },
            "speedup": 1.6825629720472908,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1516544,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1310208,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2826752,
                        "linear_total": 7077888,
                        "nnz": 2834101,
                        "total": 7087872
                    },
                    "1": {
                        "linear_attention_nnz": 1188864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1448448,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2637312,
                        "linear_total": 7077888,
                        "nnz": 2644527,
                        "total": 7087488
                    },
                    "10": {
                        "linear_attention_nnz": 746496,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 253440,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 999936,
                        "linear_total": 7077888,
                        "nnz": 1005765,
                        "total": 7086912
                    },
                    "11": {
                        "linear_attention_nnz": 486400,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 397824,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 884224,
                        "linear_total": 7077888,
                        "nnz": 890051,
                        "total": 7086912
                    },
                    "2": {
                        "linear_attention_nnz": 1540096,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1523712,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3063808,
                        "linear_total": 7077888,
                        "nnz": 3071200,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1534976,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1637376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3172352,
                        "linear_total": 7077888,
                        "nnz": 3179754,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1934336,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1545216,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3479552,
                        "linear_total": 7077888,
                        "nnz": 3487470,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1677312,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1449984,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3127296,
                        "linear_total": 7077888,
                        "nnz": 3134768,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1732608,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1130496,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2863104,
                        "linear_total": 7077888,
                        "nnz": 2870560,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1282048,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 824832,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2106880,
                        "linear_total": 7077888,
                        "nnz": 2113561,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1499136,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 463872,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1963008,
                        "linear_total": 7077888,
                        "nnz": 1969838,
                        "total": 7087680
                    },
                    "9": {
                        "linear_attention_nnz": 839680,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 213504,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1053184,
                        "linear_total": 7077888,
                        "nnz": 1059083,
                        "total": 7086912
                    }
                },
                "linear_nnz": 28177408,
                "linear_sparsity": 66.82460455246914,
                "linear_total": 84934656,
                "nnz": 52099400,
                "pruned_heads": {
                    "0": [],
                    "1": [
                        0,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        11
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887426,
                "total_sparsity": 52.15296943468937
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--1d4e4bc55a9a2bf2/checkpoint-150000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 71.5080971659919,
                "HasAns_f1": 78.18165134334741,
                "HasAns_total": 5928,
                "NoAns_exact": 72.80067283431455,
                "NoAns_f1": 72.80067283431455,
                "NoAns_total": 5945,
                "best_exact": 72.15531036806199,
                "best_exact_thresh": 0.0,
                "best_f1": 75.48730979224834,
                "best_f1_thresh": 0.0,
                "exact": 72.15531036806199,
                "f1": 75.48730979224833,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 5.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.09268295288086,
                "eval_elapsed_time": 30.314951282925904
            },
            "speedup": 1.6913343897863713,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1448960,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1261056,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2710016,
                        "linear_total": 7077888,
                        "nnz": 2717141,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1113088,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1402368,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2515456,
                        "linear_total": 7077888,
                        "nnz": 2522545,
                        "total": 7087488
                    },
                    "10": {
                        "linear_attention_nnz": 733184,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 245760,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 978944,
                        "linear_total": 7077888,
                        "nnz": 984768,
                        "total": 7086912
                    },
                    "11": {
                        "linear_attention_nnz": 475136,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 388608,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 863744,
                        "linear_total": 7077888,
                        "nnz": 869565,
                        "total": 7086912
                    },
                    "2": {
                        "linear_attention_nnz": 1537024,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1482240,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3019264,
                        "linear_total": 7077888,
                        "nnz": 3026597,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1533952,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1598976,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3132928,
                        "linear_total": 7077888,
                        "nnz": 3140305,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1928192,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1525248,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3453440,
                        "linear_total": 7077888,
                        "nnz": 3461345,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1634304,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1416192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3050496,
                        "linear_total": 7077888,
                        "nnz": 3057946,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1678336,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1110528,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2788864,
                        "linear_total": 7077888,
                        "nnz": 2796275,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1271808,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 804864,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2076672,
                        "linear_total": 7077888,
                        "nnz": 2083340,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1433600,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 456192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1889792,
                        "linear_total": 7077888,
                        "nnz": 1896585,
                        "total": 7087680
                    },
                    "9": {
                        "linear_attention_nnz": 838656,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 213504,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1052160,
                        "linear_total": 7077888,
                        "nnz": 1058059,
                        "total": 7086912
                    }
                },
                "linear_nnz": 27531776,
                "linear_sparsity": 67.58475597993827,
                "linear_total": 84934656,
                "nnz": 51453193,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        11
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887234,
                "total_sparsity": 52.74634949400956
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--1d4e4bc55a9a2bf2/checkpoint-155000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 71.9804318488529,
                "HasAns_f1": 78.86810465582569,
                "HasAns_total": 5928,
                "NoAns_exact": 71.75777964676199,
                "NoAns_f1": 71.75777964676199,
                "NoAns_total": 5945,
                "best_exact": 71.86894634885876,
                "best_exact_thresh": 0.0,
                "best_f1": 75.30785179817516,
                "best_f1_thresh": 0.0,
                "exact": 71.86894634885876,
                "f1": 75.30785179817528,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 5.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.10106954956055,
                "eval_elapsed_time": 30.318406603997573
            },
            "speedup": 1.6907909431145522,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1435648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1256448,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2692096,
                        "linear_total": 7077888,
                        "nnz": 2699218,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1107968,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1397760,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2505728,
                        "linear_total": 7077888,
                        "nnz": 2512814,
                        "total": 7087488
                    },
                    "10": {
                        "linear_attention_nnz": 747520,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 244224,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 991744,
                        "linear_total": 7077888,
                        "nnz": 997567,
                        "total": 7086912
                    },
                    "11": {
                        "linear_attention_nnz": 503808,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 387072,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 890880,
                        "linear_total": 7077888,
                        "nnz": 896700,
                        "total": 7086912
                    },
                    "2": {
                        "linear_attention_nnz": 1502208,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1476096,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2978304,
                        "linear_total": 7077888,
                        "nnz": 2985633,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1547264,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1592832,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3140096,
                        "linear_total": 7077888,
                        "nnz": 3147469,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1927168,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1525248,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3452416,
                        "linear_total": 7077888,
                        "nnz": 3460321,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1636352,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1413120,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3049472,
                        "linear_total": 7077888,
                        "nnz": 3056920,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1682432,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1107456,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2789888,
                        "linear_total": 7077888,
                        "nnz": 2797265,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1293312,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 801792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2095104,
                        "linear_total": 7077888,
                        "nnz": 2101770,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1396736,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 456192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1852928,
                        "linear_total": 7077888,
                        "nnz": 1859721,
                        "total": 7087680
                    },
                    "9": {
                        "linear_attention_nnz": 815104,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 211968,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1027072,
                        "linear_total": 7077888,
                        "nnz": 1032970,
                        "total": 7086912
                    }
                },
                "linear_nnz": 27465728,
                "linear_sparsity": 67.66251929012346,
                "linear_total": 84934656,
                "nnz": 51387090,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        11
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887234,
                "total_sparsity": 52.8070572533783
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--1d4e4bc55a9a2bf2/checkpoint-160000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 71.71052631578948,
                "HasAns_f1": 78.21541157269785,
                "HasAns_total": 5928,
                "NoAns_exact": 71.53910849453322,
                "NoAns_f1": 71.53910849453322,
                "NoAns_total": 5945,
                "best_exact": 71.6246946854207,
                "best_exact_thresh": 0.0,
                "best_f1": 74.8724804011583,
                "best_f1_thresh": 0.0,
                "exact": 71.6246946854207,
                "f1": 74.8724804011583,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 5.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.05066195678711,
                "eval_elapsed_time": 30.27150132902898
            },
            "speedup": 1.694062595154217,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1449984,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1254912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2704896,
                        "linear_total": 7077888,
                        "nnz": 2712017,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1046528,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1396224,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2442752,
                        "linear_total": 7077888,
                        "nnz": 2449837,
                        "total": 7087488
                    },
                    "10": {
                        "linear_attention_nnz": 722944,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 244224,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 967168,
                        "linear_total": 7077888,
                        "nnz": 972991,
                        "total": 7086912
                    },
                    "11": {
                        "linear_attention_nnz": 476160,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 385536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 861696,
                        "linear_total": 7077888,
                        "nnz": 867515,
                        "total": 7086912
                    },
                    "2": {
                        "linear_attention_nnz": 1516544,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1474560,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2991104,
                        "linear_total": 7077888,
                        "nnz": 2998432,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1539072,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1591296,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3130368,
                        "linear_total": 7077888,
                        "nnz": 3137740,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1909760,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1525248,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3435008,
                        "linear_total": 7077888,
                        "nnz": 3442913,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1641472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1411584,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3053056,
                        "linear_total": 7077888,
                        "nnz": 3060503,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1680384,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1104384,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2784768,
                        "linear_total": 7077888,
                        "nnz": 2792143,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1300480,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 801792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2102272,
                        "linear_total": 7077888,
                        "nnz": 2108938,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1438720,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 454656,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1893376,
                        "linear_total": 7077888,
                        "nnz": 1900168,
                        "total": 7087680
                    },
                    "9": {
                        "linear_attention_nnz": 823296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 211968,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1035264,
                        "linear_total": 7077888,
                        "nnz": 1041162,
                        "total": 7086912
                    }
                },
                "linear_nnz": 27401728,
                "linear_sparsity": 67.73787133487654,
                "linear_total": 84934656,
                "nnz": 51323081,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        11
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887234,
                "total_sparsity": 52.86584192229551
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-150000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 68.20175438596492,
                "HasAns_f1": 74.90270003478001,
                "HasAns_total": 5928,
                "NoAns_exact": 75.27333894028595,
                "NoAns_f1": 75.27333894028595,
                "NoAns_total": 5945,
                "best_exact": 71.7426092815632,
                "best_exact_thresh": 0.0,
                "best_f1": 75.08828483164983,
                "best_f1_thresh": 0.0,
                "exact": 71.7426092815632,
                "f1": 75.08828483164976,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 5,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 10.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 22.232523056030274,
                "eval_elapsed_time": 26.513782265014015
            },
            "speedup": 1.984995220235696,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1149952,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 769536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1919488,
                        "linear_total": 7077888,
                        "nnz": 1925909,
                        "total": 7087104
                    },
                    "1": {
                        "linear_attention_nnz": 760832,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 973824,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1734656,
                        "linear_total": 7077888,
                        "nnz": 1740794,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 590848,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 175104,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 765952,
                        "linear_total": 7077888,
                        "nnz": 771538,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 390144,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 264192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 654336,
                        "linear_total": 7077888,
                        "nnz": 659916,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1390592,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1055232,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2445824,
                        "linear_total": 7077888,
                        "nnz": 2452719,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1472512,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1092096,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2564608,
                        "linear_total": 7077888,
                        "nnz": 2571623,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1591296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1058304,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2649600,
                        "linear_total": 7077888,
                        "nnz": 2656881,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1536000,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1004544,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2540544,
                        "linear_total": 7077888,
                        "nnz": 2547726,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1400832,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 768000,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2168832,
                        "linear_total": 7077888,
                        "nnz": 2175764,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 1145856,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 577536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1723392,
                        "linear_total": 7077888,
                        "nnz": 1729848,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1049600,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 331776,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1381376,
                        "linear_total": 7077888,
                        "nnz": 1387608,
                        "total": 7087104
                    },
                    "9": {
                        "linear_attention_nnz": 712704,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 138240,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 850944,
                        "linear_total": 7077888,
                        "nnz": 856730,
                        "total": 7086912
                    }
                },
                "linear_nnz": 21399552,
                "linear_sparsity": 74.8046875,
                "linear_total": 84934656,
                "nnz": 45315778,
                "pruned_heads": {
                    "0": [
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        3,
                        4
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108884162,
                "total_sparsity": 58.38166252315007
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-160000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 70.4284750337382,
                "HasAns_f1": 76.73198962499656,
                "HasAns_total": 5928,
                "NoAns_exact": 72.56518082422204,
                "NoAns_f1": 72.56518082422204,
                "NoAns_total": 5945,
                "best_exact": 71.49835761812515,
                "best_exact_thresh": 0.0,
                "best_f1": 74.64560216432078,
                "best_f1_thresh": 0.0,
                "exact": 71.49835761812515,
                "f1": 74.64560216432078,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 5,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 10.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 22.191959045410158,
                "eval_elapsed_time": 26.45285324798897
            },
            "speedup": 1.9886235329515656,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1136640,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 764928,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1901568,
                        "linear_total": 7077888,
                        "nnz": 1907986,
                        "total": 7087104
                    },
                    "1": {
                        "linear_attention_nnz": 766976,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 970752,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1737728,
                        "linear_total": 7077888,
                        "nnz": 1743864,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 575488,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 175104,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 750592,
                        "linear_total": 7077888,
                        "nnz": 756178,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 363520,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 264192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 627712,
                        "linear_total": 7077888,
                        "nnz": 633260,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1369088,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1049088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2418176,
                        "linear_total": 7077888,
                        "nnz": 2425067,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1461248,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1089024,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2550272,
                        "linear_total": 7077888,
                        "nnz": 2557285,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1597440,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1056768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2654208,
                        "linear_total": 7077888,
                        "nnz": 2661488,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1525760,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 993792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2519552,
                        "linear_total": 7077888,
                        "nnz": 2526727,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1401856,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 768000,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2169856,
                        "linear_total": 7077888,
                        "nnz": 2176788,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 1100800,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 576000,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1676800,
                        "linear_total": 7077888,
                        "nnz": 1683223,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1041408,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 330240,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1371648,
                        "linear_total": 7077888,
                        "nnz": 1377879,
                        "total": 7087104
                    },
                    "9": {
                        "linear_attention_nnz": 704512,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 138240,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 842752,
                        "linear_total": 7077888,
                        "nnz": 848538,
                        "total": 7086912
                    }
                },
                "linear_nnz": 21220864,
                "linear_sparsity": 75.0150704089506,
                "linear_total": 84934656,
                "nnz": 45137005,
                "pruned_heads": {
                    "0": [
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        3,
                        4
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108884162,
                "total_sparsity": 58.54584893623005
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 71.2719298245614,
                "HasAns_f1": 78.12614467791248,
                "HasAns_total": 5928,
                "NoAns_exact": 74.61732548359966,
                "NoAns_f1": 74.61732548359966,
                "NoAns_total": 5945,
                "best_exact": 72.9470226564474,
                "best_exact_thresh": 0.0,
                "best_f1": 76.36922308183821,
                "best_f1_thresh": 0.0,
                "exact": 72.9470226564474,
                "f1": 76.36922308183819,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 5,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 5.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.076661651611328,
                "eval_elapsed_time": 30.333236434031278
            },
            "speedup": 1.6923735326861915,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1624064,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1434624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3058688,
                        "linear_total": 7077888,
                        "nnz": 3066150,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1034240,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1509888,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2544128,
                        "linear_total": 7077888,
                        "nnz": 2551127,
                        "total": 7087104
                    },
                    "10": {
                        "linear_attention_nnz": 760832,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 251904,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1012736,
                        "linear_total": 7077888,
                        "nnz": 1018692,
                        "total": 7087104
                    },
                    "11": {
                        "linear_attention_nnz": 533504,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 417792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 951296,
                        "linear_total": 7077888,
                        "nnz": 957200,
                        "total": 7087104
                    },
                    "2": {
                        "linear_attention_nnz": 1571840,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1566720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3138560,
                        "linear_total": 7077888,
                        "nnz": 3145948,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1561600,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1681920,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3243520,
                        "linear_total": 7077888,
                        "nnz": 3250951,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1954816,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1637376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3592192,
                        "linear_total": 7077888,
                        "nnz": 3600170,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1661952,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1532928,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3194880,
                        "linear_total": 7077888,
                        "nnz": 3202406,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1730560,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1167360,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2897920,
                        "linear_total": 7077888,
                        "nnz": 2905400,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1313792,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 892416,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2206208,
                        "linear_total": 7077888,
                        "nnz": 2212933,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1441792,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 473088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1914880,
                        "linear_total": 7077888,
                        "nnz": 1921620,
                        "total": 7087488
                    },
                    "9": {
                        "linear_attention_nnz": 906240,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 222720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1128960,
                        "linear_total": 7077888,
                        "nnz": 1134961,
                        "total": 7086912
                    }
                },
                "linear_nnz": 28883968,
                "linear_sparsity": 65.99271797839505,
                "linear_total": 84934656,
                "nnz": 52806280,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        3,
                        11,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887042,
                "total_sparsity": 51.50361417660698
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-160000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 70.39473684210526,
                "HasAns_f1": 77.14341107371645,
                "HasAns_total": 5928,
                "NoAns_exact": 74.02859545836837,
                "NoAns_f1": 74.02859545836837,
                "NoAns_total": 5945,
                "best_exact": 72.21426766613324,
                "best_exact_thresh": 0.0,
                "best_f1": 75.58377333824572,
                "best_f1_thresh": 0.0,
                "exact": 72.21426766613324,
                "f1": 75.58377333824575,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 5,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 5.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 25.73425926208496,
                "eval_elapsed_time": 30.011940322001465
            },
            "speedup": 1.7148910932524941,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1526784,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1353216,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2880000,
                        "linear_total": 7077888,
                        "nnz": 2887377,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 974848,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1456128,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2430976,
                        "linear_total": 7077888,
                        "nnz": 2437844,
                        "total": 7086912
                    },
                    "10": {
                        "linear_attention_nnz": 769024,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 245760,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1014784,
                        "linear_total": 7077888,
                        "nnz": 1020736,
                        "total": 7087104
                    },
                    "11": {
                        "linear_attention_nnz": 494592,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 403968,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 898560,
                        "linear_total": 7077888,
                        "nnz": 904423,
                        "total": 7087104
                    },
                    "2": {
                        "linear_attention_nnz": 1556480,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1522176,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3078656,
                        "linear_total": 7077888,
                        "nnz": 3086015,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1564672,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1635840,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3200512,
                        "linear_total": 7077888,
                        "nnz": 3207913,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1926144,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1585152,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3511296,
                        "linear_total": 7077888,
                        "nnz": 3519240,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1669120,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1482240,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3151360,
                        "linear_total": 7077888,
                        "nnz": 3158853,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1571840,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1128960,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2700800,
                        "linear_total": 7077888,
                        "nnz": 2708063,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 1311744,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 870912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2182656,
                        "linear_total": 7077888,
                        "nnz": 2189367,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1420288,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 460800,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1881088,
                        "linear_total": 7077888,
                        "nnz": 1887788,
                        "total": 7087488
                    },
                    "9": {
                        "linear_attention_nnz": 858112,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 216576,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1074688,
                        "linear_total": 7077888,
                        "nnz": 1080621,
                        "total": 7086912
                    }
                },
                "linear_nnz": 28005376,
                "linear_sparsity": 67.02715084876543,
                "linear_total": 84934656,
                "nnz": 51926962,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        3,
                        11,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108886658,
                "total_sparsity": 52.31099663284734
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--489923cf6100d0c5/checkpoint-140000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 69.26450742240216,
                "HasAns_f1": 76.47931310832281,
                "HasAns_total": 5928,
                "NoAns_exact": 71.15222876366694,
                "NoAns_f1": 71.15222876366694,
                "NoAns_total": 5945,
                "best_exact": 70.2097195317106,
                "best_exact_thresh": 0.0,
                "best_f1": 73.81195722278605,
                "best_f1_thresh": 0.0,
                "exact": 70.2097195317106,
                "f1": 73.81195722278603,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 5,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 20.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 20.692826644897462,
                "eval_elapsed_time": 24.999257856048644
            },
            "speedup": 2.1326932640631844,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 957440,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 419328,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1376768,
                        "linear_total": 7077888,
                        "nnz": 1382833,
                        "total": 7086912
                    },
                    "1": {
                        "linear_attention_nnz": 748544,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 588288,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1336832,
                        "linear_total": 7077888,
                        "nnz": 1342719,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 463872,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 127488,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 591360,
                        "linear_total": 7077888,
                        "nnz": 596851,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 324608,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 162816,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 487424,
                        "linear_total": 7077888,
                        "nnz": 492842,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1277952,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 645120,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1923072,
                        "linear_total": 7077888,
                        "nnz": 1929668,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1306624,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 688128,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1994752,
                        "linear_total": 7077888,
                        "nnz": 2001408,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1461248,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 746496,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2207744,
                        "linear_total": 7077888,
                        "nnz": 2214694,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1142784,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 660480,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1803264,
                        "linear_total": 7077888,
                        "nnz": 1809806,
                        "total": 7087104
                    },
                    "6": {
                        "linear_attention_nnz": 1047552,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 486912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1534464,
                        "linear_total": 7077888,
                        "nnz": 1540861,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 941056,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 391680,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1332736,
                        "linear_total": 7077888,
                        "nnz": 1338911,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 935936,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 227328,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1163264,
                        "linear_total": 7077888,
                        "nnz": 1169428,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 556032,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 105984,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 662016,
                        "linear_total": 7077888,
                        "nnz": 667621,
                        "total": 7086720
                    }
                },
                "linear_nnz": 16413696,
                "linear_sparsity": 80.67491319444444,
                "linear_total": 84934656,
                "nnz": 40326364,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2,
                        11,
                        6
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108883394,
                "total_sparsity": 62.96371510976228
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--489923cf6100d0c5/checkpoint-155000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 68.3029689608637,
                "HasAns_f1": 74.93622452026204,
                "HasAns_total": 5928,
                "NoAns_exact": 72.63246425567704,
                "NoAns_f1": 72.63246425567704,
                "NoAns_total": 5945,
                "best_exact": 70.47081613745473,
                "best_exact_thresh": 0.0,
                "best_f1": 73.78269510284802,
                "best_f1_thresh": 0.0,
                "exact": 70.47081613745473,
                "f1": 73.78269510284804,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 5,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 20.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 20.5203953704834,
                "eval_elapsed_time": 24.813339828979224
            },
            "speedup": 2.1506141184530403,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 935936,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 402432,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1338368,
                        "linear_total": 7077888,
                        "nnz": 1344326,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 718848,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 569856,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1288704,
                        "linear_total": 7077888,
                        "nnz": 1294579,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 464896,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 122880,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 587776,
                        "linear_total": 7077888,
                        "nnz": 593296,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 316416,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 156672,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 473088,
                        "linear_total": 7077888,
                        "nnz": 478502,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1244160,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 631296,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1875456,
                        "linear_total": 7077888,
                        "nnz": 1882043,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1270784,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 680448,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1951232,
                        "linear_total": 7077888,
                        "nnz": 1957851,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1404928,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 734208,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2139136,
                        "linear_total": 7077888,
                        "nnz": 2146078,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1121280,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 652800,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1774080,
                        "linear_total": 7077888,
                        "nnz": 1780617,
                        "total": 7087104
                    },
                    "6": {
                        "linear_attention_nnz": 1036288,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 473088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1509376,
                        "linear_total": 7077888,
                        "nnz": 1515732,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 925696,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 388608,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1314304,
                        "linear_total": 7077888,
                        "nnz": 1320413,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 918528,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 227328,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1145856,
                        "linear_total": 7077888,
                        "nnz": 1151988,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 528384,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 105984,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 634368,
                        "linear_total": 7077888,
                        "nnz": 639941,
                        "total": 7086720
                    }
                },
                "linear_nnz": 16031744,
                "linear_sparsity": 81.12461419753086,
                "linear_total": 84934656,
                "nnz": 39944088,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2,
                        11,
                        6
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108883202,
                "total_sparsity": 63.31473793358868
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--489923cf6100d0c5/checkpoint-160000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 69.73684210526316,
                "HasAns_f1": 77.18943506323211,
                "HasAns_total": 5928,
                "NoAns_exact": 68.19175777964676,
                "NoAns_f1": 68.19175777964676,
                "NoAns_total": 5945,
                "best_exact": 68.96319380106124,
                "best_exact_thresh": 0.0,
                "best_f1": 72.68415489386358,
                "best_f1_thresh": 0.0,
                "exact": 68.96319380106124,
                "f1": 72.68415489386355,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 5,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 20.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 20.526136947631837,
                "eval_elapsed_time": 24.803041459061205
            },
            "speedup": 2.1500125480304555,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 912384,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 400896,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1313280,
                        "linear_total": 7077888,
                        "nnz": 1319237,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 740352,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 568320,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1308672,
                        "linear_total": 7077888,
                        "nnz": 1314546,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 468992,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 122880,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 591872,
                        "linear_total": 7077888,
                        "nnz": 597296,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 297984,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 155136,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 453120,
                        "linear_total": 7077888,
                        "nnz": 458533,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1220608,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 629760,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1850368,
                        "linear_total": 7077888,
                        "nnz": 1856890,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1288192,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 680448,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1968640,
                        "linear_total": 7077888,
                        "nnz": 1975259,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1471488,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 734208,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2205696,
                        "linear_total": 7077888,
                        "nnz": 2212638,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1118208,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 652800,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1771008,
                        "linear_total": 7077888,
                        "nnz": 1777545,
                        "total": 7087104
                    },
                    "6": {
                        "linear_attention_nnz": 998400,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 473088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1471488,
                        "linear_total": 7077888,
                        "nnz": 1477844,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 930816,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 388608,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1319424,
                        "linear_total": 7077888,
                        "nnz": 1325533,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 894976,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 225792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1120768,
                        "linear_total": 7077888,
                        "nnz": 1126899,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 527360,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 105984,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 633344,
                        "linear_total": 7077888,
                        "nnz": 638949,
                        "total": 7086720
                    }
                },
                "linear_nnz": 16007680,
                "linear_sparsity": 81.15294656635803,
                "linear_total": 84934656,
                "nnz": 39919891,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2,
                        11,
                        6
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108883202,
                "total_sparsity": 63.336960828907294
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 66.95344129554655,
                "HasAns_f1": 73.95621625172882,
                "HasAns_total": 5928,
                "NoAns_exact": 73.86038687973087,
                "NoAns_f1": 73.86038687973087,
                "NoAns_total": 5945,
                "best_exact": 70.41185883938347,
                "best_exact_thresh": 0.0,
                "best_f1": 73.90823296051964,
                "best_f1_thresh": 0.0,
                "exact": 70.41185883938347,
                "f1": 73.90823296051956,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 20.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 20.151872497558593,
                "eval_elapsed_time": 24.383727533044294
            },
            "speedup": 2.189942994396503,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 924672,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 396288,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1320960,
                        "linear_total": 7077888,
                        "nnz": 1326914,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 711680,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 560640,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1272320,
                        "linear_total": 7077888,
                        "nnz": 1278125,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 423936,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 118272,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 542208,
                        "linear_total": 7077888,
                        "nnz": 547629,
                        "total": 7086528
                    },
                    "11": {
                        "linear_attention_nnz": 314368,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 152064,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 466432,
                        "linear_total": 7077888,
                        "nnz": 471875,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1217536,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 599040,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1816576,
                        "linear_total": 7077888,
                        "nnz": 1823078,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1281024,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 646656,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1927680,
                        "linear_total": 7077888,
                        "nnz": 1934245,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1203200,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 672768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1875968,
                        "linear_total": 7077888,
                        "nnz": 1882582,
                        "total": 7087296
                    },
                    "5": {
                        "linear_attention_nnz": 979968,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 609792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1589760,
                        "linear_total": 7077888,
                        "nnz": 1596077,
                        "total": 7086912
                    },
                    "6": {
                        "linear_attention_nnz": 1017856,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 486912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1504768,
                        "linear_total": 7077888,
                        "nnz": 1511069,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 886784,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 374784,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1261568,
                        "linear_total": 7077888,
                        "nnz": 1267636,
                        "total": 7086912
                    },
                    "8": {
                        "linear_attention_nnz": 891904,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 224256,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1116160,
                        "linear_total": 7077888,
                        "nnz": 1122322,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 510976,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 101376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 612352,
                        "linear_total": 7077888,
                        "nnz": 617890,
                        "total": 7086720
                    }
                },
                "linear_nnz": 15306752,
                "linear_sparsity": 81.97820216049382,
                "linear_total": 84934656,
                "nnz": 39218164,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108882242,
                "total_sparsity": 63.98112008016881
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-160000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.4.2",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 68.31983805668017,
                "HasAns_f1": 75.3064164201114,
                "HasAns_total": 5928,
                "NoAns_exact": 70.58031959629942,
                "NoAns_f1": 70.58031959629942,
                "NoAns_total": 5945,
                "best_exact": 69.45169712793734,
                "best_exact_thresh": 0.0,
                "best_f1": 72.93998454800146,
                "best_f1_thresh": 0.0,
                "exact": 69.45169712793734,
                "f1": 72.93998454800146,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "regularization": "l1",
                "regularization_final_lambda": 20.0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 19.807459594726563,
                "eval_elapsed_time": 24.067060329020023
            },
            "speedup": 2.2280218111235897,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 920576,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 396288,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1316864,
                        "linear_total": 7077888,
                        "nnz": 1322818,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 698368,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 559104,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1257472,
                        "linear_total": 7077888,
                        "nnz": 1263276,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 433152,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 118272,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 551424,
                        "linear_total": 7077888,
                        "nnz": 556877,
                        "total": 7086528
                    },
                    "11": {
                        "linear_attention_nnz": 291840,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 152064,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 443904,
                        "linear_total": 7077888,
                        "nnz": 449347,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1227776,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 599040,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1826816,
                        "linear_total": 7077888,
                        "nnz": 1833318,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1266688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 643584,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1910272,
                        "linear_total": 7077888,
                        "nnz": 1916835,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1227776,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 672768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1900544,
                        "linear_total": 7077888,
                        "nnz": 1907158,
                        "total": 7087296
                    },
                    "5": {
                        "linear_attention_nnz": 965632,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 609792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1575424,
                        "linear_total": 7077888,
                        "nnz": 1581741,
                        "total": 7086912
                    },
                    "6": {
                        "linear_attention_nnz": 967680,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 486912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1454592,
                        "linear_total": 7077888,
                        "nnz": 1460893,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 862208,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 374784,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1236992,
                        "linear_total": 7077888,
                        "nnz": 1243060,
                        "total": 7086912
                    },
                    "8": {
                        "linear_attention_nnz": 893952,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 224256,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1118208,
                        "linear_total": 7077888,
                        "nnz": 1124370,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 500736,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 101376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 602112,
                        "linear_total": 7077888,
                        "nnz": 607650,
                        "total": 7086720
                    }
                },
                "linear_nnz": 15194624,
                "linear_sparsity": 82.11021894290124,
                "linear_total": 84934656,
                "nnz": 39106065,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108882242,
                "total_sparsity": 64.08407442602073
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_1___es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv--1c40a6fda898dfc6/checkpoint-10000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 67.3076923076923,
                "HasAns_f1": 72.81932879612465,
                "HasAns_total": 5928,
                "NoAns_exact": 74.24726661059714,
                "NoAns_f1": 74.24726661059714,
                "NoAns_total": 5945,
                "best_exact": 70.78244757011707,
                "best_exact_thresh": 0.0,
                "best_f1": 73.53431997839023,
                "best_f1_thresh": 0.0,
                "exact": 70.78244757011707,
                "f1": 73.53431997839029,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 20,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 45.2769192199707,
                "eval_elapsed_time": 49.82686016801745
            },
            "speedup": 0.9747008577503776,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 2142208,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4700160,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6842368,
                        "linear_total": 7077888,
                        "nnz": 6852340,
                        "total": 7087872
                    },
                    "1": {
                        "linear_attention_nnz": 2319360,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4694016,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7013376,
                        "linear_total": 7077888,
                        "nnz": 7023344,
                        "total": 7087872
                    },
                    "10": {
                        "linear_attention_nnz": 2316288,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4681728,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6998016,
                        "linear_total": 7077888,
                        "nnz": 7007976,
                        "total": 7087872
                    },
                    "11": {
                        "linear_attention_nnz": 2337792,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4618752,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6956544,
                        "linear_total": 7077888,
                        "nnz": 6966463,
                        "total": 7087872
                    },
                    "2": {
                        "linear_attention_nnz": 2323456,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4698624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7022080,
                        "linear_total": 7077888,
                        "nnz": 7032051,
                        "total": 7087872
                    },
                    "3": {
                        "linear_attention_nnz": 2319360,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4709376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7028736,
                        "linear_total": 7077888,
                        "nnz": 7038714,
                        "total": 7087872
                    },
                    "4": {
                        "linear_attention_nnz": 2331648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4703232,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7034880,
                        "linear_total": 7077888,
                        "nnz": 7044854,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 2340864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4701696,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7042560,
                        "linear_total": 7077888,
                        "nnz": 7052533,
                        "total": 7087872
                    },
                    "6": {
                        "linear_attention_nnz": 2329600,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4695552,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7025152,
                        "linear_total": 7077888,
                        "nnz": 7035121,
                        "total": 7087872
                    },
                    "7": {
                        "linear_attention_nnz": 2342912,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4675584,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7018496,
                        "linear_total": 7077888,
                        "nnz": 7028452,
                        "total": 7087872
                    },
                    "8": {
                        "linear_attention_nnz": 2318336,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4678656,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6996992,
                        "linear_total": 7077888,
                        "nnz": 7006950,
                        "total": 7087872
                    },
                    "9": {
                        "linear_attention_nnz": 2322432,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4694016,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7016448,
                        "linear_total": 7077888,
                        "nnz": 7026416,
                        "total": 7087872
                    }
                },
                "linear_nnz": 83995648,
                "linear_sparsity": 1.1055652006172867,
                "linear_total": 84934656,
                "nnz": 107953936,
                "pruned_heads": {
                    "0": [],
                    "1": [],
                    "10": [],
                    "11": [],
                    "2": [],
                    "3": [],
                    "4": [],
                    "5": [],
                    "6": [],
                    "7": [],
                    "8": [],
                    "9": []
                },
                "total": 108893186,
                "total_sparsity": 0.862542491869045
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_1___es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv--1c40a6fda898dfc6/checkpoint-15000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 63.37719298245614,
                "HasAns_f1": 68.81261065959998,
                "HasAns_total": 5928,
                "NoAns_exact": 75.74432296047098,
                "NoAns_f1": 75.74432296047098,
                "NoAns_total": 5945,
                "best_exact": 69.58645666638591,
                "best_exact_thresh": 0.0,
                "best_f1": 72.3002742348278,
                "best_f1_thresh": 0.0,
                "exact": 69.56961172407985,
                "f1": 72.28342929252172,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 20,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 43.87494717407227,
                "eval_elapsed_time": 48.39579511305783
            },
            "speedup": 1.005846270877776,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1837056,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4647936,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6484992,
                        "linear_total": 7077888,
                        "nnz": 6494930,
                        "total": 7087872
                    },
                    "1": {
                        "linear_attention_nnz": 2066432,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4588032,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6654464,
                        "linear_total": 7077888,
                        "nnz": 6664363,
                        "total": 7087872
                    },
                    "10": {
                        "linear_attention_nnz": 1779712,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 3030528,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 4810240,
                        "linear_total": 7077888,
                        "nnz": 4819125,
                        "total": 7087872
                    },
                    "11": {
                        "linear_attention_nnz": 1807360,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 3159552,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 4966912,
                        "linear_total": 7077888,
                        "nnz": 4975881,
                        "total": 7087872
                    },
                    "2": {
                        "linear_attention_nnz": 2165760,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4595712,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6761472,
                        "linear_total": 7077888,
                        "nnz": 6771376,
                        "total": 7087872
                    },
                    "3": {
                        "linear_attention_nnz": 2159616,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4631040,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6790656,
                        "linear_total": 7077888,
                        "nnz": 6800583,
                        "total": 7087872
                    },
                    "4": {
                        "linear_attention_nnz": 2233344,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4620288,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6853632,
                        "linear_total": 7077888,
                        "nnz": 6863552,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 2254848,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4600320,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6855168,
                        "linear_total": 7077888,
                        "nnz": 6865075,
                        "total": 7087872
                    },
                    "6": {
                        "linear_attention_nnz": 2196480,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4525056,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6721536,
                        "linear_total": 7077888,
                        "nnz": 6731394,
                        "total": 7087872
                    },
                    "7": {
                        "linear_attention_nnz": 2216960,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4488192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6705152,
                        "linear_total": 7077888,
                        "nnz": 6714986,
                        "total": 7087872
                    },
                    "8": {
                        "linear_attention_nnz": 2136064,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4334592,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6470656,
                        "linear_total": 7077888,
                        "nnz": 6480390,
                        "total": 7087872
                    },
                    "9": {
                        "linear_attention_nnz": 2029568,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4018176,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6047744,
                        "linear_total": 7077888,
                        "nnz": 6057272,
                        "total": 7087872
                    }
                },
                "linear_nnz": 76122624,
                "linear_sparsity": 10.375072337962965,
                "linear_total": 84934656,
                "nnz": 100077649,
                "pruned_heads": {
                    "0": [],
                    "1": [],
                    "10": [],
                    "11": [],
                    "2": [],
                    "3": [],
                    "4": [],
                    "5": [],
                    "6": [],
                    "7": [],
                    "8": [],
                    "9": []
                },
                "total": 108893186,
                "total_sparsity": 8.095581848436318
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_1___es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv--4ecf729f2e1956d9/checkpoint-150000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 71.05263157894737,
                "HasAns_f1": 77.88524114101236,
                "HasAns_total": 5928,
                "NoAns_exact": 73.6080740117746,
                "NoAns_f1": 73.6080740117746,
                "NoAns_total": 5945,
                "best_exact": 72.33218226227575,
                "best_exact_thresh": 0.0,
                "best_f1": 75.7435955094686,
                "best_f1_thresh": 0.0,
                "exact": 72.33218226227575,
                "f1": 75.7435955094686,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 10,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 22.351992279052734,
                "eval_elapsed_time": 26.97958334197756
            },
            "speedup": 1.9743856139999647,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1049600,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 724992,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1774592,
                        "linear_total": 7077888,
                        "nnz": 1780888,
                        "total": 7086912
                    },
                    "1": {
                        "linear_attention_nnz": 770048,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 964608,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1734656,
                        "linear_total": 7077888,
                        "nnz": 1740788,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 606208,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 221184,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 827392,
                        "linear_total": 7077888,
                        "nnz": 833200,
                        "total": 7086912
                    },
                    "11": {
                        "linear_attention_nnz": 412672,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 278016,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 690688,
                        "linear_total": 7077888,
                        "nnz": 696181,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1387520,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 993792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2381312,
                        "linear_total": 7077888,
                        "nnz": 2388167,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1432576,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 992256,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2424832,
                        "linear_total": 7077888,
                        "nnz": 2431814,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1619968,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 998400,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2618368,
                        "linear_total": 7077888,
                        "nnz": 2625610,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1348608,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 958464,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2307072,
                        "linear_total": 7077888,
                        "nnz": 2314032,
                        "total": 7087296
                    },
                    "6": {
                        "linear_attention_nnz": 1351680,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 685056,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2036736,
                        "linear_total": 7077888,
                        "nnz": 2043550,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 1101824,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 494592,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1596416,
                        "linear_total": 7077888,
                        "nnz": 1602818,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1088512,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 253440,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1341952,
                        "linear_total": 7077888,
                        "nnz": 1348229,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 662528,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 133632,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 796160,
                        "linear_total": 7077888,
                        "nnz": 801975,
                        "total": 7087104
                    }
                },
                "linear_nnz": 20530176,
                "linear_sparsity": 75.82826967592592,
                "linear_total": 84934656,
                "nnz": 44445974,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2,
                        6
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        7
                    ]
                },
                "total": 108884354,
                "total_sparsity": 59.18056877115696
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_1___es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv--4ecf729f2e1956d9/checkpoint-160000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 70.90080971659918,
                "HasAns_f1": 77.78317738891889,
                "HasAns_total": 5928,
                "NoAns_exact": 73.06980656013457,
                "NoAns_f1": 73.06980656013457,
                "NoAns_total": 5945,
                "best_exact": 71.98686094500127,
                "best_exact_thresh": 0.0,
                "best_f1": 75.42311762499034,
                "best_f1_thresh": 0.0,
                "exact": 71.98686094500127,
                "f1": 75.42311762499034,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 10,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 22.405832763671874,
                "eval_elapsed_time": 26.99683353002183
            },
            "speedup": 1.9696412298298225,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1067008,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 721920,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1788928,
                        "linear_total": 7077888,
                        "nnz": 1795222,
                        "total": 7086912
                    },
                    "1": {
                        "linear_attention_nnz": 759808,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 960000,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1719808,
                        "linear_total": 7077888,
                        "nnz": 1725937,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 601088,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 221184,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 822272,
                        "linear_total": 7077888,
                        "nnz": 828080,
                        "total": 7086912
                    },
                    "11": {
                        "linear_attention_nnz": 406528,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 274944,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 681472,
                        "linear_total": 7077888,
                        "nnz": 686963,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1364992,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 989184,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2354176,
                        "linear_total": 7077888,
                        "nnz": 2361028,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1391616,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 990720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2382336,
                        "linear_total": 7077888,
                        "nnz": 2389317,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1574912,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 989184,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2564096,
                        "linear_total": 7077888,
                        "nnz": 2571332,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1344512,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 955392,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2299904,
                        "linear_total": 7077888,
                        "nnz": 2306862,
                        "total": 7087296
                    },
                    "6": {
                        "linear_attention_nnz": 1349632,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 683520,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2033152,
                        "linear_total": 7077888,
                        "nnz": 2039965,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 1083392,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 493056,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1576448,
                        "linear_total": 7077888,
                        "nnz": 1582849,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1079296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 251904,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1331200,
                        "linear_total": 7077888,
                        "nnz": 1337476,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 668672,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 133632,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 802304,
                        "linear_total": 7077888,
                        "nnz": 808119,
                        "total": 7087104
                    }
                },
                "linear_nnz": 20356096,
                "linear_sparsity": 76.03322723765432,
                "linear_total": 84934656,
                "nnz": 44271872,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2,
                        6
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        7
                    ]
                },
                "total": 108884354,
                "total_sparsity": 59.340465022183075
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_1___es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2--52d8e88d43486bb/checkpoint-155000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 69.34885290148448,
                "HasAns_f1": 76.44255068347434,
                "HasAns_total": 5928,
                "NoAns_exact": 72.27922624053826,
                "NoAns_f1": 72.27922624053826,
                "NoAns_total": 5945,
                "best_exact": 70.81613745472922,
                "best_exact_thresh": 0.0,
                "best_f1": 74.35790789620458,
                "best_f1_thresh": 0.0,
                "exact": 70.81613745472922,
                "f1": 74.35790789620452,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 20,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 20.218405868530272,
                "eval_elapsed_time": 24.83853198797442
            },
            "speedup": 2.1827364771962623,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 873472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 434688,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1308160,
                        "linear_total": 7077888,
                        "nnz": 1314107,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 695296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 542208,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1237504,
                        "linear_total": 7077888,
                        "nnz": 1243329,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 465920,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 153600,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 619520,
                        "linear_total": 7077888,
                        "nnz": 625092,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 340992,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 185856,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 526848,
                        "linear_total": 7077888,
                        "nnz": 532377,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1183744,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 626688,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1810432,
                        "linear_total": 7077888,
                        "nnz": 1817016,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1261568,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 642048,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1903616,
                        "linear_total": 7077888,
                        "nnz": 1910178,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1212416,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 672768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1885184,
                        "linear_total": 7077888,
                        "nnz": 1891766,
                        "total": 7087296
                    },
                    "5": {
                        "linear_attention_nnz": 967680,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 583680,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1551360,
                        "linear_total": 7077888,
                        "nnz": 1557660,
                        "total": 7086912
                    },
                    "6": {
                        "linear_attention_nnz": 1113088,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 459264,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1572352,
                        "linear_total": 7077888,
                        "nnz": 1578923,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 885760,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 354816,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1240576,
                        "linear_total": 7077888,
                        "nnz": 1246663,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 857088,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 172032,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1029120,
                        "linear_total": 7077888,
                        "nnz": 1035216,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 431104,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 90624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 521728,
                        "linear_total": 7077888,
                        "nnz": 527035,
                        "total": 7086528
                    }
                },
                "linear_nnz": 15206400,
                "linear_sparsity": 82.09635416666666,
                "linear_total": 84934656,
                "nnz": 39118084,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        2,
                        4,
                        5,
                        6,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        3,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108882626,
                "total_sparsity": 64.07316259988072
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_1___es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2--52d8e88d43486bb/checkpoint-160000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 70.29352226720648,
                "HasAns_f1": 77.11347931083279,
                "HasAns_total": 5928,
                "NoAns_exact": 71.05130361648445,
                "NoAns_f1": 71.05130361648445,
                "NoAns_total": 5945,
                "best_exact": 70.6729554451276,
                "best_exact_thresh": 0.0,
                "best_f1": 74.07805149116632,
                "best_f1_thresh": 0.0,
                "exact": 70.6729554451276,
                "f1": 74.07805149116633,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 20,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 19.951193420410156,
                "eval_elapsed_time": 24.477091429987922
            },
            "speedup": 2.2119705358002966,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 894976,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 434688,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1329664,
                        "linear_total": 7077888,
                        "nnz": 1335611,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 700416,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 542208,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1242624,
                        "linear_total": 7077888,
                        "nnz": 1248449,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 458752,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 153600,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 612352,
                        "linear_total": 7077888,
                        "nnz": 617924,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 344064,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 185856,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 529920,
                        "linear_total": 7077888,
                        "nnz": 535449,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1242112,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 625152,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1867264,
                        "linear_total": 7077888,
                        "nnz": 1873847,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1238016,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 642048,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1880064,
                        "linear_total": 7077888,
                        "nnz": 1886626,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1199104,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 672768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1871872,
                        "linear_total": 7077888,
                        "nnz": 1878454,
                        "total": 7087296
                    },
                    "5": {
                        "linear_attention_nnz": 953344,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 582144,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1535488,
                        "linear_total": 7077888,
                        "nnz": 1541787,
                        "total": 7086912
                    },
                    "6": {
                        "linear_attention_nnz": 1056768,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 457728,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1514496,
                        "linear_total": 7077888,
                        "nnz": 1521034,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 865280,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 354816,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1220096,
                        "linear_total": 7077888,
                        "nnz": 1226183,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 846848,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 172032,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1018880,
                        "linear_total": 7077888,
                        "nnz": 1024976,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 420864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 90624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 511488,
                        "linear_total": 7077888,
                        "nnz": 516827,
                        "total": 7086528
                    }
                },
                "linear_nnz": 15134208,
                "linear_sparsity": 82.18135127314815,
                "linear_total": 84934656,
                "nnz": 39045889,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        2,
                        4,
                        5,
                        6,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        3,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108882626,
                "total_sparsity": 64.13946794413279
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_1___es-steps_pdtbs32_nte20_ws2700_ls250_ss2500_stl50_est2500_rn-__data_2to__devel_data__nn_pruning__o--5d25482d33f511d2/checkpoint-5000": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 65.9919028340081,
                "HasAns_f1": 72.0320111205815,
                "HasAns_total": 5928,
                "NoAns_exact": 72.51471825063078,
                "NoAns_f1": 72.51471825063078,
                "NoAns_total": 5945,
                "best_exact": 69.26640276257054,
                "best_exact_thresh": 0.0,
                "best_f1": 72.28213273164387,
                "best_f1_thresh": 0.0,
                "exact": 69.2579802914175,
                "f1": 72.2737102604908,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 20,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 45.08802215576172,
                "eval_elapsed_time": 49.63239961198997
            },
            "speedup": 0.9787843841883963,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 2326528,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4718592,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7045120,
                        "linear_total": 7077888,
                        "nnz": 7055104,
                        "total": 7087872
                    },
                    "1": {
                        "linear_attention_nnz": 2355200,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4718592,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7073792,
                        "linear_total": 7077888,
                        "nnz": 7083776,
                        "total": 7087872
                    },
                    "10": {
                        "linear_attention_nnz": 2357248,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4718592,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7075840,
                        "linear_total": 7077888,
                        "nnz": 7085824,
                        "total": 7087872
                    },
                    "11": {
                        "linear_attention_nnz": 2358272,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4715520,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7073792,
                        "linear_total": 7077888,
                        "nnz": 7083774,
                        "total": 7087872
                    },
                    "2": {
                        "linear_attention_nnz": 2355200,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4715520,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7070720,
                        "linear_total": 7077888,
                        "nnz": 7080702,
                        "total": 7087872
                    },
                    "3": {
                        "linear_attention_nnz": 2357248,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4718592,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7075840,
                        "linear_total": 7077888,
                        "nnz": 7085824,
                        "total": 7087872
                    },
                    "4": {
                        "linear_attention_nnz": 2354176,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4717056,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7071232,
                        "linear_total": 7077888,
                        "nnz": 7081215,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 2357248,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4718592,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7075840,
                        "linear_total": 7077888,
                        "nnz": 7085824,
                        "total": 7087872
                    },
                    "6": {
                        "linear_attention_nnz": 2352128,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4715520,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7067648,
                        "linear_total": 7077888,
                        "nnz": 7077630,
                        "total": 7087872
                    },
                    "7": {
                        "linear_attention_nnz": 2358272,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4712448,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7070720,
                        "linear_total": 7077888,
                        "nnz": 7080700,
                        "total": 7087872
                    },
                    "8": {
                        "linear_attention_nnz": 2356224,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4706304,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7062528,
                        "linear_total": 7077888,
                        "nnz": 7072504,
                        "total": 7087872
                    },
                    "9": {
                        "linear_attention_nnz": 2358272,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4713984,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7072256,
                        "linear_total": 7077888,
                        "nnz": 7082237,
                        "total": 7087872
                    }
                },
                "linear_nnz": 84835328,
                "linear_sparsity": 0.11694637345679437,
                "linear_total": 84934656,
                "nnz": 108793836,
                "pruned_heads": {
                    "0": [],
                    "1": [],
                    "10": [],
                    "11": [],
                    "2": [],
                    "3": [],
                    "4": [],
                    "5": [],
                    "6": [],
                    "7": [],
                    "8": [],
                    "9": []
                },
                "total": 108893186,
                "total_sparsity": 0.0912361954401808
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 8,
                "per_device_train_batch_size": 32,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 2700,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_1___es-steps_pdtbs32_nte20_ws2700_ls250_ss2500_stl50_est2500_rn-__data_2to__devel_data__nn_pruning__o--5d25482d33f511d2/checkpoint-7500": {
            "config": {
                "_name_or_path": "bert-base-uncased",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 75.15182186234817,
                "HasAns_f1": 82.66072173363392,
                "HasAns_total": 5928,
                "NoAns_exact": 59.73086627417998,
                "NoAns_f1": 59.73086627417998,
                "NoAns_total": 5945,
                "best_exact": 67.43872652236166,
                "best_exact_thresh": 0.0,
                "best_f1": 71.18780076113731,
                "best_f1_thresh": 0.0,
                "exact": 67.43030405120862,
                "f1": 71.17937828998426,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-base-uncased",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 20,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 45.250162994384766,
                "eval_elapsed_time": 49.752105894032866
            },
            "speedup": 0.9752771941501385,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 2202624,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4707840,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6910464,
                        "linear_total": 7077888,
                        "nnz": 6920441,
                        "total": 7087872
                    },
                    "1": {
                        "linear_attention_nnz": 2317312,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4697088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7014400,
                        "linear_total": 7077888,
                        "nnz": 7024370,
                        "total": 7087872
                    },
                    "10": {
                        "linear_attention_nnz": 2290688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4423680,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6714368,
                        "linear_total": 7077888,
                        "nnz": 6724160,
                        "total": 7087872
                    },
                    "11": {
                        "linear_attention_nnz": 2315264,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4111872,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6427136,
                        "linear_total": 7077888,
                        "nnz": 6436725,
                        "total": 7087872
                    },
                    "2": {
                        "linear_attention_nnz": 2337792,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4698624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7036416,
                        "linear_total": 7077888,
                        "nnz": 7046387,
                        "total": 7087872
                    },
                    "3": {
                        "linear_attention_nnz": 2319360,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4698624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7017984,
                        "linear_total": 7077888,
                        "nnz": 7027955,
                        "total": 7087872
                    },
                    "4": {
                        "linear_attention_nnz": 2337792,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4697088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7034880,
                        "linear_total": 7077888,
                        "nnz": 7044850,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 2343936,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4694016,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7037952,
                        "linear_total": 7077888,
                        "nnz": 7047920,
                        "total": 7087872
                    },
                    "6": {
                        "linear_attention_nnz": 2317312,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4677120,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6994432,
                        "linear_total": 7077888,
                        "nnz": 7004389,
                        "total": 7087872
                    },
                    "7": {
                        "linear_attention_nnz": 2348032,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4669440,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 7017472,
                        "linear_total": 7077888,
                        "nnz": 7027424,
                        "total": 7087872
                    },
                    "8": {
                        "linear_attention_nnz": 2323456,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4654080,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6977536,
                        "linear_total": 7077888,
                        "nnz": 6987478,
                        "total": 7087872
                    },
                    "9": {
                        "linear_attention_nnz": 2317312,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 4643328,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 6960640,
                        "linear_total": 7077888,
                        "nnz": 6970575,
                        "total": 7087872
                    }
                },
                "linear_nnz": 83143680,
                "linear_sparsity": 2.108651620370372,
                "linear_total": 84934656,
                "nnz": 107101396,
                "pruned_heads": {
                    "0": [],
                    "1": [],
                    "10": [],
                    "11": [],
                    "2": [],
                    "3": [],
                    "4": [],
                    "5": [],
                    "6": [],
                    "7": [],
                    "8": [],
                    "9": []
                },
                "total": 108893186,
                "total_sparsity": 1.6454564934852756
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 8,
                "per_device_train_batch_size": 32,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_1/",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 2700,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_2/hp_mnop-bert-large-uncased-whole-word-masking_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_2___es-steps_pdtbs8_pdebs128_nte20_ls250_stl50_est5000_rn---58ebffa395d47d66/checkpoint-325000": {
            "config": {
                "_name_or_path": "bert-large-uncased-whole-word-masking",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 1024,
                "initializer_range": 0.02,
                "intermediate_size": 4096,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 16,
                "num_hidden_layers": 24,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 74.8144399460189,
                "HasAns_f1": 80.555306012496,
                "HasAns_total": 5928,
                "NoAns_exact": 84.57527333894029,
                "NoAns_f1": 84.57527333894029,
                "NoAns_total": 5945,
                "best_exact": 79.70184452118251,
                "best_exact_thresh": 0.0,
                "best_f1": 82.56816761071966,
                "best_f1_thresh": 0.0,
                "exact": 79.70184452118251,
                "f1": 82.56816761071981,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-large-uncased-whole-word-masking",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 5,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 52.126098907470706,
                "eval_elapsed_time": 56.62768604200005
            },
            "speedup": 0.8466287123910415,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 998400,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 587776,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 1586176,
                        "linear_total": 12582912,
                        "nnz": 1593727,
                        "total": 12594496
                    },
                    "1": {
                        "linear_attention_nnz": 1025024,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 624640,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 1649664,
                        "linear_total": 12582912,
                        "nnz": 1657297,
                        "total": 12594496
                    },
                    "10": {
                        "linear_attention_nnz": 1531904,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1267712,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2799616,
                        "linear_total": 12582912,
                        "nnz": 2807915,
                        "total": 12594688
                    },
                    "11": {
                        "linear_attention_nnz": 1508352,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1351680,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2860032,
                        "linear_total": 12582912,
                        "nnz": 2868180,
                        "total": 12594496
                    },
                    "12": {
                        "linear_attention_nnz": 1525760,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1236992,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2762752,
                        "linear_total": 12582912,
                        "nnz": 2771132,
                        "total": 12594880
                    },
                    "13": {
                        "linear_attention_nnz": 1813504,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1423360,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3236864,
                        "linear_total": 12582912,
                        "nnz": 3245559,
                        "total": 12595264
                    },
                    "14": {
                        "linear_attention_nnz": 1774592,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1153024,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2927616,
                        "linear_total": 12582912,
                        "nnz": 2936051,
                        "total": 12595072
                    },
                    "15": {
                        "linear_attention_nnz": 1909760,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1077248,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2987008,
                        "linear_total": 12582912,
                        "nnz": 2996110,
                        "total": 12595840
                    },
                    "16": {
                        "linear_attention_nnz": 2169856,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1091584,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3261440,
                        "linear_total": 12582912,
                        "nnz": 3270645,
                        "total": 12595840
                    },
                    "17": {
                        "linear_attention_nnz": 1823744,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1071104,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2894848,
                        "linear_total": 12582912,
                        "nnz": 2903531,
                        "total": 12595264
                    },
                    "18": {
                        "linear_attention_nnz": 1886208,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 774144,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2660352,
                        "linear_total": 12582912,
                        "nnz": 2669146,
                        "total": 12595648
                    },
                    "19": {
                        "linear_attention_nnz": 1472512,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 446464,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 1918976,
                        "linear_total": 12582912,
                        "nnz": 1927354,
                        "total": 12595648
                    },
                    "2": {
                        "linear_attention_nnz": 595968,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 876544,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 1472512,
                        "linear_total": 12582912,
                        "nnz": 1479660,
                        "total": 12593728
                    },
                    "20": {
                        "linear_attention_nnz": 1079296,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 299008,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 1378304,
                        "linear_total": 12582912,
                        "nnz": 1386290,
                        "total": 12595456
                    },
                    "21": {
                        "linear_attention_nnz": 512000,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 155648,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 667648,
                        "linear_total": 12582912,
                        "nnz": 674764,
                        "total": 12594688
                    },
                    "22": {
                        "linear_attention_nnz": 375808,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 90112,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 465920,
                        "linear_total": 12582912,
                        "nnz": 472716,
                        "total": 12594304
                    },
                    "23": {
                        "linear_attention_nnz": 343040,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 221184,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 564224,
                        "linear_total": 12582912,
                        "nnz": 571244,
                        "total": 12594688
                    },
                    "3": {
                        "linear_attention_nnz": 916480,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1085440,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2001920,
                        "linear_total": 12582912,
                        "nnz": 2009554,
                        "total": 12594112
                    },
                    "4": {
                        "linear_attention_nnz": 678912,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1146880,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 1825792,
                        "linear_total": 12582912,
                        "nnz": 1833264,
                        "total": 12593920
                    },
                    "5": {
                        "linear_attention_nnz": 509952,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1308672,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 1818624,
                        "linear_total": 12582912,
                        "nnz": 1825983,
                        "total": 12593920
                    },
                    "6": {
                        "linear_attention_nnz": 717824,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1441792,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2159616,
                        "linear_total": 12582912,
                        "nnz": 2167168,
                        "total": 12593920
                    },
                    "7": {
                        "linear_attention_nnz": 1009664,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1468416,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2478080,
                        "linear_total": 12582912,
                        "nnz": 2485901,
                        "total": 12594112
                    },
                    "8": {
                        "linear_attention_nnz": 1327104,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1468416,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2795520,
                        "linear_total": 12582912,
                        "nnz": 2803661,
                        "total": 12594496
                    },
                    "9": {
                        "linear_attention_nnz": 631808,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1531904,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2163712,
                        "linear_total": 12582912,
                        "nnz": 2171276,
                        "total": 12593920
                    }
                },
                "linear_nnz": 51337216,
                "linear_sparsity": 83.00035264756944,
                "linear_total": 301989888,
                "nnz": 83313090,
                "pruned_heads": {
                    "0": [
                        0,
                        1,
                        3,
                        4,
                        5,
                        8,
                        9,
                        13,
                        15
                    ],
                    "1": [
                        0,
                        1,
                        3,
                        5,
                        7,
                        9,
                        10,
                        13,
                        14
                    ],
                    "10": [
                        1,
                        2,
                        4,
                        5,
                        6,
                        8,
                        11,
                        13
                    ],
                    "11": [
                        0,
                        2,
                        5,
                        6,
                        7,
                        8,
                        10,
                        12,
                        15
                    ],
                    "12": [
                        0,
                        2,
                        6,
                        8,
                        9,
                        11,
                        13
                    ],
                    "13": [
                        2,
                        6,
                        10,
                        12,
                        15
                    ],
                    "14": [
                        1,
                        5,
                        6,
                        10,
                        11,
                        15
                    ],
                    "15": [
                        0,
                        9
                    ],
                    "16": [
                        5,
                        7
                    ],
                    "17": [
                        1,
                        4,
                        8,
                        12,
                        14
                    ],
                    "18": [
                        3,
                        11,
                        4
                    ],
                    "19": [
                        0,
                        12,
                        5
                    ],
                    "2": [
                        0,
                        1,
                        4,
                        6,
                        7,
                        8,
                        9,
                        10,
                        11,
                        12,
                        13,
                        14,
                        15
                    ],
                    "20": [
                        0,
                        10,
                        4,
                        12
                    ],
                    "21": [
                        0,
                        2,
                        3,
                        4,
                        8,
                        11,
                        12,
                        15
                    ],
                    "22": [
                        0,
                        1,
                        3,
                        4,
                        7,
                        9,
                        10,
                        11,
                        13,
                        15
                    ],
                    "23": [
                        2,
                        4,
                        8,
                        9,
                        10,
                        13,
                        14,
                        15
                    ],
                    "3": [
                        0,
                        3,
                        5,
                        6,
                        7,
                        8,
                        9,
                        11,
                        13,
                        14,
                        15
                    ],
                    "4": [
                        0,
                        2,
                        3,
                        4,
                        5,
                        6,
                        7,
                        9,
                        10,
                        11,
                        12,
                        14
                    ],
                    "5": [
                        1,
                        3,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9,
                        12,
                        13,
                        14,
                        15
                    ],
                    "6": [
                        0,
                        1,
                        2,
                        3,
                        4,
                        7,
                        8,
                        9,
                        10,
                        11,
                        12,
                        15
                    ],
                    "7": [
                        0,
                        3,
                        4,
                        5,
                        8,
                        9,
                        10,
                        11,
                        12,
                        13,
                        14
                    ],
                    "8": [
                        3,
                        4,
                        5,
                        7,
                        8,
                        9,
                        10,
                        11,
                        12
                    ],
                    "9": [
                        0,
                        1,
                        2,
                        3,
                        5,
                        6,
                        7,
                        9,
                        10,
                        13,
                        14,
                        15
                    ]
                },
                "total": 334057858,
                "total_sparsity": 75.06028132408129
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_2/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_2/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 8,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_2/",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_3/hp_mnop-bert-large-uncased-whole-word-masking_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_3___es-steps_pdtbs8_pdebs128_nte20.06_ls250_stl50_est5000_--128cd5d9b5683065/checkpoint-330000": {
            "config": {
                "_name_or_path": "bert-large-uncased-whole-word-masking",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 1024,
                "initializer_range": 0.02,
                "intermediate_size": 4096,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 16,
                "num_hidden_layers": 24,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 76.48448043184885,
                "HasAns_f1": 82.55514100819374,
                "HasAns_total": 5928,
                "NoAns_exact": 83.8856181665265,
                "NoAns_f1": 83.8856181665265,
                "NoAns_total": 5945,
                "best_exact": 80.19034784805862,
                "best_exact_thresh": 0.0,
                "best_f1": 83.22133208932635,
                "best_f1_thresh": 0.0,
                "exact": 80.19034784805862,
                "f1": 83.22133208932645,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "bert-large-uncased-whole-word-masking",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 32,
                "attention_block_rows": 32,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "sigmoied_threshold",
                "bias_mask": true,
                "decoder_attention_lambda": null,
                "decoder_dense_lambda": null,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "sigmoied_threshold:1d_alt",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "madlag/bert-large-uncased-whole-word-masking-finetuned-squadv2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": false,
                "final_threshold": 0.1,
                "final_warmup": 10,
                "gelu_patch": 0,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 0,
                "initial_warmup": 1,
                "layer_norm_patch": 0,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "l1",
                "regularization_final_lambda": 2,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 63.65266278076172,
                "eval_elapsed_time": 68.118653698
            },
            "speedup": 0.6933166669240776,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1329152,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1294336,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2623488,
                        "linear_total": 12582912,
                        "nnz": 2631960,
                        "total": 12594880
                    },
                    "1": {
                        "linear_attention_nnz": 1539072,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1429504,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2968576,
                        "linear_total": 12582912,
                        "nnz": 2977018,
                        "total": 12594880
                    },
                    "10": {
                        "linear_attention_nnz": 1718272,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1990656,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3708928,
                        "linear_total": 12582912,
                        "nnz": 3717580,
                        "total": 12594688
                    },
                    "11": {
                        "linear_attention_nnz": 2091008,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2207744,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 4298752,
                        "linear_total": 12582912,
                        "nnz": 4307894,
                        "total": 12595072
                    },
                    "12": {
                        "linear_attention_nnz": 2011136,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1988608,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3999744,
                        "linear_total": 12582912,
                        "nnz": 4008939,
                        "total": 12595264
                    },
                    "13": {
                        "linear_attention_nnz": 2132992,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2392064,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 4525056,
                        "linear_total": 12582912,
                        "nnz": 4534288,
                        "total": 12595072
                    },
                    "14": {
                        "linear_attention_nnz": 2490368,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1941504,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 4431872,
                        "linear_total": 12582912,
                        "nnz": 4441620,
                        "total": 12595840
                    },
                    "15": {
                        "linear_attention_nnz": 2515968,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1873920,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 4389888,
                        "linear_total": 12582912,
                        "nnz": 4399635,
                        "total": 12595840
                    },
                    "16": {
                        "linear_attention_nnz": 2995200,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1984512,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 4979712,
                        "linear_total": 12582912,
                        "nnz": 4989801,
                        "total": 12596224
                    },
                    "17": {
                        "linear_attention_nnz": 2373632,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1820672,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 4194304,
                        "linear_total": 12582912,
                        "nnz": 4203737,
                        "total": 12595648
                    },
                    "18": {
                        "linear_attention_nnz": 2628608,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1259520,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3888128,
                        "linear_total": 12582912,
                        "nnz": 3897575,
                        "total": 12595840
                    },
                    "19": {
                        "linear_attention_nnz": 2135040,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 776192,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2911232,
                        "linear_total": 12582912,
                        "nnz": 2920315,
                        "total": 12596032
                    },
                    "2": {
                        "linear_attention_nnz": 761856,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 1775616,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2537472,
                        "linear_total": 12582912,
                        "nnz": 2545251,
                        "total": 12593920
                    },
                    "20": {
                        "linear_attention_nnz": 1545216,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 507904,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2053120,
                        "linear_total": 12582912,
                        "nnz": 2061560,
                        "total": 12595648
                    },
                    "21": {
                        "linear_attention_nnz": 608256,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 260096,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 868352,
                        "linear_total": 12582912,
                        "nnz": 875551,
                        "total": 12594496
                    },
                    "22": {
                        "linear_attention_nnz": 617472,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 149504,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 766976,
                        "linear_total": 12582912,
                        "nnz": 774281,
                        "total": 12594688
                    },
                    "23": {
                        "linear_attention_nnz": 527360,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 284672,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 812032,
                        "linear_total": 12582912,
                        "nnz": 819435,
                        "total": 12594880
                    },
                    "3": {
                        "linear_attention_nnz": 1164288,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2027520,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3191808,
                        "linear_total": 12582912,
                        "nnz": 3200094,
                        "total": 12594304
                    },
                    "4": {
                        "linear_attention_nnz": 835584,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2035712,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2871296,
                        "linear_total": 12582912,
                        "nnz": 2879362,
                        "total": 12594112
                    },
                    "5": {
                        "linear_attention_nnz": 734208,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2123776,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 2857984,
                        "linear_total": 12582912,
                        "nnz": 2866029,
                        "total": 12594112
                    },
                    "6": {
                        "linear_attention_nnz": 816128,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2312192,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3128320,
                        "linear_total": 12582912,
                        "nnz": 3136361,
                        "total": 12593920
                    },
                    "7": {
                        "linear_attention_nnz": 1503232,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2338816,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3842048,
                        "linear_total": 12582912,
                        "nnz": 3850774,
                        "total": 12594688
                    },
                    "8": {
                        "linear_attention_nnz": 1784832,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2328576,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 4113408,
                        "linear_total": 12582912,
                        "nnz": 4122385,
                        "total": 12594880
                    },
                    "9": {
                        "linear_attention_nnz": 1396736,
                        "linear_attention_total": 4194304,
                        "linear_dense_nnz": 2463744,
                        "linear_dense_total": 8388608,
                        "linear_nnz": 3860480,
                        "linear_total": 12582912,
                        "nnz": 3869363,
                        "total": 12594688
                    }
                },
                "linear_nnz": 77822976,
                "linear_sparsity": 74.22993977864583,
                "linear_total": 301989888,
                "nnz": 109815770,
                "pruned_heads": {
                    "0": [
                        0,
                        3,
                        4,
                        5,
                        9,
                        13,
                        15
                    ],
                    "1": [
                        0,
                        1,
                        7,
                        9,
                        10,
                        13,
                        14
                    ],
                    "10": [
                        1,
                        2,
                        4,
                        5,
                        6,
                        8,
                        11,
                        13
                    ],
                    "11": [
                        2,
                        5,
                        6,
                        8,
                        12,
                        15
                    ],
                    "12": [
                        0,
                        2,
                        6,
                        9,
                        13
                    ],
                    "13": [
                        2,
                        3,
                        6,
                        10,
                        12,
                        15
                    ],
                    "14": [
                        10,
                        5
                    ],
                    "15": [
                        0,
                        9
                    ],
                    "16": [],
                    "17": [
                        1,
                        4,
                        12
                    ],
                    "18": [
                        11,
                        4
                    ],
                    "19": [
                        5
                    ],
                    "2": [
                        0,
                        1,
                        4,
                        6,
                        7,
                        8,
                        9,
                        11,
                        12,
                        13,
                        14,
                        15
                    ],
                    "20": [
                        0,
                        1,
                        12
                    ],
                    "21": [
                        0,
                        2,
                        3,
                        4,
                        8,
                        10,
                        11,
                        12,
                        15
                    ],
                    "22": [
                        0,
                        1,
                        3,
                        7,
                        9,
                        10,
                        11,
                        13
                    ],
                    "23": [
                        2,
                        4,
                        7,
                        8,
                        9,
                        10,
                        13
                    ],
                    "3": [
                        0,
                        3,
                        5,
                        6,
                        7,
                        8,
                        9,
                        13,
                        14,
                        15
                    ],
                    "4": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        9,
                        10,
                        11,
                        12,
                        14
                    ],
                    "5": [
                        3,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9,
                        12,
                        13,
                        14,
                        15
                    ],
                    "6": [
                        0,
                        1,
                        2,
                        3,
                        4,
                        7,
                        8,
                        9,
                        10,
                        11,
                        12,
                        15
                    ],
                    "7": [
                        3,
                        5,
                        8,
                        9,
                        10,
                        11,
                        12,
                        13
                    ],
                    "8": [
                        3,
                        5,
                        7,
                        8,
                        9,
                        11,
                        12
                    ],
                    "9": [
                        0,
                        1,
                        2,
                        5,
                        6,
                        7,
                        13,
                        14
                    ]
                },
                "total": 334064578,
                "total_sparsity": 67.12738277806874
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 5000,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_3/",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 20.06,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_3/",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 8,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_3/",
                "save_steps": 5000,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 5400,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-60000": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        3,
                        4
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 71.84547908232119,
                "HasAns_f1": 78.27031796244835,
                "HasAns_total": 5928,
                "NoAns_exact": 73.00252312867956,
                "NoAns_f1": 73.00252312867956,
                "NoAns_total": 5945,
                "best_exact": 72.42482944495914,
                "best_exact_thresh": 0.0,
                "best_f1": 75.63264927831177,
                "best_f1_thresh": 0.0,
                "exact": 72.42482944495914,
                "f1": 75.63264927831169,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-150000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 22.437649841308595,
                "eval_elapsed_time": 26.890222856076434
            },
            "speedup": 1.9668482355381207,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 769536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2342400,
                        "linear_total": 7077888,
                        "nnz": 2349045,
                        "total": 7087104
                    },
                    "1": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 973824,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1956864,
                        "linear_total": 7077888,
                        "nnz": 1963066,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 175104,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1354752,
                        "linear_total": 7077888,
                        "nnz": 1360626,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 264192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1443840,
                        "linear_total": 7077888,
                        "nnz": 1449772,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1055232,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2824704,
                        "linear_total": 7077888,
                        "nnz": 2831727,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1092096,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2861568,
                        "linear_total": 7077888,
                        "nnz": 2868615,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1058304,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3220992,
                        "linear_total": 7077888,
                        "nnz": 3228401,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1004544,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2970624,
                        "linear_total": 7077888,
                        "nnz": 2977806,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 768000,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2734080,
                        "linear_total": 7077888,
                        "nnz": 2741108,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 577536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2150400,
                        "linear_total": 7077888,
                        "nnz": 2156920,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 331776,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1904640,
                        "linear_total": 7077888,
                        "nnz": 1911000,
                        "total": 7087104
                    },
                    "9": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 138240,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1514496,
                        "linear_total": 7077888,
                        "nnz": 1520538,
                        "total": 7086912
                    }
                },
                "linear_nnz": 27279360,
                "linear_sparsity": 67.88194444444444,
                "linear_total": 84934656,
                "nnz": 51197346,
                "pruned_heads": {
                    "0": [
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        11,
                        3,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        3,
                        4
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108884162,
                "total_sparsity": 52.97998803535816
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-80000": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        3,
                        4
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 71.94669365721997,
                "HasAns_f1": 78.3671658213592,
                "HasAns_total": 5928,
                "NoAns_exact": 72.54835996635829,
                "NoAns_f1": 72.54835996635829,
                "NoAns_total": 5945,
                "best_exact": 72.24795755074538,
                "best_exact_thresh": 0.0,
                "best_f1": 75.4535971522799,
                "best_f1_thresh": 0.0,
                "exact": 72.24795755074538,
                "f1": 75.45359715227984,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-150000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 22.43720558166504,
                "eval_elapsed_time": 26.837573747034185
            },
            "speedup": 1.9668871793938012,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 769536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2342400,
                        "linear_total": 7077888,
                        "nnz": 2349045,
                        "total": 7087104
                    },
                    "1": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 973824,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1956864,
                        "linear_total": 7077888,
                        "nnz": 1963066,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 175104,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1354752,
                        "linear_total": 7077888,
                        "nnz": 1360626,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 264192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1443840,
                        "linear_total": 7077888,
                        "nnz": 1449772,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1055232,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2824704,
                        "linear_total": 7077888,
                        "nnz": 2831727,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1092096,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2861568,
                        "linear_total": 7077888,
                        "nnz": 2868615,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1058304,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3220992,
                        "linear_total": 7077888,
                        "nnz": 3228401,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1004544,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2970624,
                        "linear_total": 7077888,
                        "nnz": 2977806,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 768000,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2734080,
                        "linear_total": 7077888,
                        "nnz": 2741108,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 577536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2150400,
                        "linear_total": 7077888,
                        "nnz": 2156920,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 331776,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1904640,
                        "linear_total": 7077888,
                        "nnz": 1911000,
                        "total": 7087104
                    },
                    "9": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 138240,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1514496,
                        "linear_total": 7077888,
                        "nnz": 1520538,
                        "total": 7086912
                    }
                },
                "linear_nnz": 27279360,
                "linear_sparsity": 67.88194444444444,
                "linear_total": 84934656,
                "nnz": 51197346,
                "pruned_heads": {
                    "0": [
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        11,
                        3,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        3,
                        4
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108884162,
                "total_sparsity": 52.97998803535816
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-82350": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        3,
                        4
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 72.13225371120107,
                "HasAns_f1": 78.56627936550444,
                "HasAns_total": 5928,
                "NoAns_exact": 72.27922624053826,
                "NoAns_f1": 72.27922624053826,
                "NoAns_total": 5945,
                "best_exact": 72.20584519498021,
                "best_exact_thresh": 0.0,
                "best_f1": 75.418251838517,
                "best_f1_thresh": 0.0,
                "exact": 72.20584519498021,
                "f1": 75.41825183851692,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c/checkpoint-150000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 22.428810897827148,
                "eval_elapsed_time": 26.791726927971467
            },
            "speedup": 1.967623348426169,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 769536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2342400,
                        "linear_total": 7077888,
                        "nnz": 2349045,
                        "total": 7087104
                    },
                    "1": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 973824,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1956864,
                        "linear_total": 7077888,
                        "nnz": 1963066,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 175104,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1354752,
                        "linear_total": 7077888,
                        "nnz": 1360626,
                        "total": 7086720
                    },
                    "11": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 264192,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1443840,
                        "linear_total": 7077888,
                        "nnz": 1449772,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1055232,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2824704,
                        "linear_total": 7077888,
                        "nnz": 2831727,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1092096,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2861568,
                        "linear_total": 7077888,
                        "nnz": 2868615,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1058304,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3220992,
                        "linear_total": 7077888,
                        "nnz": 3228401,
                        "total": 7087680
                    },
                    "5": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1004544,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2970624,
                        "linear_total": 7077888,
                        "nnz": 2977806,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 768000,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2734080,
                        "linear_total": 7077888,
                        "nnz": 2741108,
                        "total": 7087488
                    },
                    "7": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 577536,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2150400,
                        "linear_total": 7077888,
                        "nnz": 2156920,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 331776,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1904640,
                        "linear_total": 7077888,
                        "nnz": 1911000,
                        "total": 7087104
                    },
                    "9": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 138240,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1514496,
                        "linear_total": 7077888,
                        "nnz": 1520538,
                        "total": 7086912
                    }
                },
                "linear_nnz": 27279360,
                "linear_sparsity": 67.88194444444444,
                "linear_total": 84934656,
                "nnz": 51197346,
                "pruned_heads": {
                    "0": [
                        2,
                        4,
                        5,
                        6
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        2
                    ],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        2,
                        3
                    ],
                    "7": [
                        11,
                        3,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8,
                        3,
                        4
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108884162,
                "total_sparsity": 52.97998803535816
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--333fe38360923d6c",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-62500": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        3,
                        11,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 72.60458839406208,
                "HasAns_f1": 78.74976951058292,
                "HasAns_total": 5928,
                "NoAns_exact": 75.23969722455845,
                "NoAns_f1": 75.23969722455845,
                "NoAns_total": 5945,
                "best_exact": 73.92402931019961,
                "best_exact_thresh": 0.0,
                "best_f1": 76.99222047155187,
                "best_f1_thresh": 0.0,
                "exact": 73.92402931019961,
                "f1": 76.9922204715519,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.13464465332031,
                "eval_elapsed_time": 30.81871720601339
            },
            "speedup": 1.6886187887920359,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1434624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3597312,
                        "linear_total": 7077888,
                        "nnz": 3604966,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1509888,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3082752,
                        "linear_total": 7077888,
                        "nnz": 3089879,
                        "total": 7087104
                    },
                    "10": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 251904,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1824768,
                        "linear_total": 7077888,
                        "nnz": 1831076,
                        "total": 7087104
                    },
                    "11": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 417792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1990656,
                        "linear_total": 7077888,
                        "nnz": 1997072,
                        "total": 7087104
                    },
                    "2": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1566720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3532800,
                        "linear_total": 7077888,
                        "nnz": 3540348,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1681920,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3451392,
                        "linear_total": 7077888,
                        "nnz": 3458823,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 2359296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1637376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3996672,
                        "linear_total": 7077888,
                        "nnz": 4004650,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1532928,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3499008,
                        "linear_total": 7077888,
                        "nnz": 3506534,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1167360,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3330048,
                        "linear_total": 7077888,
                        "nnz": 3337528,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 892416,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2465280,
                        "linear_total": 7077888,
                        "nnz": 2472005,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 473088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2439168,
                        "linear_total": 7077888,
                        "nnz": 2446004,
                        "total": 7087488
                    },
                    "9": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 222720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1598976,
                        "linear_total": 7077888,
                        "nnz": 1605073,
                        "total": 7086912
                    }
                },
                "linear_nnz": 34808832,
                "linear_sparsity": 59.01692708333333,
                "linear_total": 84934656,
                "nnz": 58732680,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        11,
                        3,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        11,
                        3,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887042,
                "total_sparsity": 46.060909616775156
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-70000": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        3,
                        11,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 72.52024291497976,
                "HasAns_f1": 78.86454940318372,
                "HasAns_total": 5928,
                "NoAns_exact": 74.41547518923466,
                "NoAns_f1": 74.41547518923466,
                "NoAns_total": 5945,
                "best_exact": 73.46921586793565,
                "best_exact_thresh": 0.0,
                "best_f1": 76.6368271592751,
                "best_f1_thresh": 0.0,
                "exact": 73.46921586793565,
                "f1": 76.6368271592751,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.144072219848635,
                "eval_elapsed_time": 30.55245121801272
            },
            "speedup": 1.6880098719470071,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1434624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3597312,
                        "linear_total": 7077888,
                        "nnz": 3604966,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1509888,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3082752,
                        "linear_total": 7077888,
                        "nnz": 3089879,
                        "total": 7087104
                    },
                    "10": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 251904,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1824768,
                        "linear_total": 7077888,
                        "nnz": 1831076,
                        "total": 7087104
                    },
                    "11": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 417792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1990656,
                        "linear_total": 7077888,
                        "nnz": 1997072,
                        "total": 7087104
                    },
                    "2": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1566720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3532800,
                        "linear_total": 7077888,
                        "nnz": 3540348,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1681920,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3451392,
                        "linear_total": 7077888,
                        "nnz": 3458823,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 2359296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1637376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3996672,
                        "linear_total": 7077888,
                        "nnz": 4004650,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1532928,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3499008,
                        "linear_total": 7077888,
                        "nnz": 3506534,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1167360,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3330048,
                        "linear_total": 7077888,
                        "nnz": 3337528,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 892416,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2465280,
                        "linear_total": 7077888,
                        "nnz": 2472005,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 473088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2439168,
                        "linear_total": 7077888,
                        "nnz": 2446004,
                        "total": 7087488
                    },
                    "9": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 222720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1598976,
                        "linear_total": 7077888,
                        "nnz": 1605073,
                        "total": 7086912
                    }
                },
                "linear_nnz": 34808832,
                "linear_sparsity": 59.01692708333333,
                "linear_total": 84934656,
                "nnz": 58732680,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        11,
                        3,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        11,
                        3,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887042,
                "total_sparsity": 46.060909616775156
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-77500": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        3,
                        11,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 72.35155195681511,
                "HasAns_f1": 79.01066967291747,
                "HasAns_total": 5928,
                "NoAns_exact": 73.82674516400337,
                "NoAns_f1": 73.82674516400337,
                "NoAns_total": 5945,
                "best_exact": 73.09020466604902,
                "best_exact_thresh": 0.0,
                "best_f1": 76.41499619481645,
                "best_f1_thresh": 0.0,
                "exact": 73.09020466604902,
                "f1": 76.41499619481642,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.154957778930665,
                "eval_elapsed_time": 30.775646347901784
            },
            "speedup": 1.6873073309088056,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1434624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3597312,
                        "linear_total": 7077888,
                        "nnz": 3604966,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1509888,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3082752,
                        "linear_total": 7077888,
                        "nnz": 3089879,
                        "total": 7087104
                    },
                    "10": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 251904,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1824768,
                        "linear_total": 7077888,
                        "nnz": 1831076,
                        "total": 7087104
                    },
                    "11": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 417792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1990656,
                        "linear_total": 7077888,
                        "nnz": 1997072,
                        "total": 7087104
                    },
                    "2": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1566720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3532800,
                        "linear_total": 7077888,
                        "nnz": 3540348,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1681920,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3451392,
                        "linear_total": 7077888,
                        "nnz": 3458823,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 2359296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1637376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3996672,
                        "linear_total": 7077888,
                        "nnz": 4004650,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1532928,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3499008,
                        "linear_total": 7077888,
                        "nnz": 3506534,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1167360,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3330048,
                        "linear_total": 7077888,
                        "nnz": 3337528,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 892416,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2465280,
                        "linear_total": 7077888,
                        "nnz": 2472005,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 473088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2439168,
                        "linear_total": 7077888,
                        "nnz": 2446004,
                        "total": 7087488
                    },
                    "9": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 222720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1598976,
                        "linear_total": 7077888,
                        "nnz": 1605073,
                        "total": 7086912
                    }
                },
                "linear_nnz": 34808832,
                "linear_sparsity": 59.01692708333333,
                "linear_total": 84934656,
                "nnz": 58732680,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        11,
                        3,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        11,
                        3,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887042,
                "total_sparsity": 46.060909616775156
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-80000": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        3,
                        11,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 72.52024291497976,
                "HasAns_f1": 79.00733071830157,
                "HasAns_total": 5928,
                "NoAns_exact": 73.82674516400337,
                "NoAns_f1": 73.82674516400337,
                "NoAns_total": 5945,
                "best_exact": 73.17442937757939,
                "best_exact_thresh": 0.0,
                "best_f1": 76.41332910789967,
                "best_f1_thresh": 0.0,
                "exact": 73.17442937757939,
                "f1": 76.41332910789964,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.134426086425783,
                "eval_elapsed_time": 30.504467725055292
            },
            "speedup": 1.6886329110139469,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1434624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3597312,
                        "linear_total": 7077888,
                        "nnz": 3604966,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1509888,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3082752,
                        "linear_total": 7077888,
                        "nnz": 3089879,
                        "total": 7087104
                    },
                    "10": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 251904,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1824768,
                        "linear_total": 7077888,
                        "nnz": 1831076,
                        "total": 7087104
                    },
                    "11": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 417792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1990656,
                        "linear_total": 7077888,
                        "nnz": 1997072,
                        "total": 7087104
                    },
                    "2": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1566720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3532800,
                        "linear_total": 7077888,
                        "nnz": 3540348,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1681920,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3451392,
                        "linear_total": 7077888,
                        "nnz": 3458823,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 2359296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1637376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3996672,
                        "linear_total": 7077888,
                        "nnz": 4004650,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1532928,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3499008,
                        "linear_total": 7077888,
                        "nnz": 3506534,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1167360,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3330048,
                        "linear_total": 7077888,
                        "nnz": 3337528,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 892416,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2465280,
                        "linear_total": 7077888,
                        "nnz": 2472005,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 473088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2439168,
                        "linear_total": 7077888,
                        "nnz": 2446004,
                        "total": 7087488
                    },
                    "9": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 222720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1598976,
                        "linear_total": 7077888,
                        "nnz": 1605073,
                        "total": 7086912
                    }
                },
                "linear_nnz": 34808832,
                "linear_sparsity": 59.01692708333333,
                "linear_total": 84934656,
                "nnz": 58732680,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        11,
                        3,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        11,
                        3,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887042,
                "total_sparsity": 46.060909616775156
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-82350": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        3,
                        11,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        3,
                        11,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 72.31781376518218,
                "HasAns_f1": 78.78776667488243,
                "HasAns_total": 5928,
                "NoAns_exact": 74.02859545836837,
                "NoAns_f1": 74.02859545836837,
                "NoAns_total": 5945,
                "best_exact": 73.17442937757939,
                "best_exact_thresh": 0.0,
                "best_f1": 76.40477392813138,
                "best_f1_thresh": 0.0,
                "exact": 73.17442937757939,
                "f1": 76.40477392813135,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 26.140411270141602,
                "eval_elapsed_time": 30.49465876701288
            },
            "speedup": 1.6882462767679685,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1434624,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3597312,
                        "linear_total": 7077888,
                        "nnz": 3604966,
                        "total": 7087680
                    },
                    "1": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1509888,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3082752,
                        "linear_total": 7077888,
                        "nnz": 3089879,
                        "total": 7087104
                    },
                    "10": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 251904,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1824768,
                        "linear_total": 7077888,
                        "nnz": 1831076,
                        "total": 7087104
                    },
                    "11": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 417792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1990656,
                        "linear_total": 7077888,
                        "nnz": 1997072,
                        "total": 7087104
                    },
                    "2": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1566720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3532800,
                        "linear_total": 7077888,
                        "nnz": 3540348,
                        "total": 7087488
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1681920,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3451392,
                        "linear_total": 7077888,
                        "nnz": 3458823,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 2359296,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1637376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3996672,
                        "linear_total": 7077888,
                        "nnz": 4004650,
                        "total": 7087872
                    },
                    "5": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1532928,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3499008,
                        "linear_total": 7077888,
                        "nnz": 3506534,
                        "total": 7087488
                    },
                    "6": {
                        "linear_attention_nnz": 2162688,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 1167360,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 3330048,
                        "linear_total": 7077888,
                        "nnz": 3337528,
                        "total": 7087680
                    },
                    "7": {
                        "linear_attention_nnz": 1572864,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 892416,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2465280,
                        "linear_total": 7077888,
                        "nnz": 2472005,
                        "total": 7087104
                    },
                    "8": {
                        "linear_attention_nnz": 1966080,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 473088,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2439168,
                        "linear_total": 7077888,
                        "nnz": 2446004,
                        "total": 7087488
                    },
                    "9": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 222720,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1598976,
                        "linear_total": 7077888,
                        "nnz": 1605073,
                        "total": 7086912
                    }
                },
                "linear_nnz": 34808832,
                "linear_sparsity": 59.01692708333333,
                "linear_total": 84934656,
                "nnz": 58732680,
                "pruned_heads": {
                    "0": [
                        4
                    ],
                    "1": [
                        0,
                        8,
                        2,
                        5
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6
                    ],
                    "11": [
                        11,
                        3,
                        5,
                        7
                    ],
                    "2": [
                        8,
                        4
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [],
                    "5": [
                        1,
                        2
                    ],
                    "6": [
                        3
                    ],
                    "7": [
                        11,
                        3,
                        6,
                        7
                    ],
                    "8": [
                        0,
                        8
                    ],
                    "9": [
                        1,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108887042,
                "total_sparsity": 46.060909616775156
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-65000": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 70.41160593792172,
                "HasAns_f1": 77.2515244087712,
                "HasAns_total": 5928,
                "NoAns_exact": 73.15391084945333,
                "NoAns_f1": 73.15391084945333,
                "NoAns_total": 5945,
                "best_exact": 71.78472163732839,
                "best_exact_thresh": 0.0,
                "best_f1": 75.1997841063923,
                "best_f1_thresh": 0.0,
                "exact": 71.78472163732839,
                "f1": 75.19978410639239,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 19.93420053100586,
                "eval_elapsed_time": 24.44789553899318
            },
            "speedup": 2.2138561278822038,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 396288,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1575936,
                        "linear_total": 7077888,
                        "nnz": 1581954,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 560640,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1543680,
                        "linear_total": 7077888,
                        "nnz": 1549613,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 118272,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1101312,
                        "linear_total": 7077888,
                        "nnz": 1106957,
                        "total": 7086528
                    },
                    "11": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 152064,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1331712,
                        "linear_total": 7077888,
                        "nnz": 1337571,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 599040,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2368512,
                        "linear_total": 7077888,
                        "nnz": 2375238,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 646656,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2416128,
                        "linear_total": 7077888,
                        "nnz": 2422885,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 672768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2442240,
                        "linear_total": 7077888,
                        "nnz": 2449014,
                        "total": 7087296
                    },
                    "5": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 609792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1986048,
                        "linear_total": 7077888,
                        "nnz": 1992397,
                        "total": 7086912
                    },
                    "6": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 486912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2256384,
                        "linear_total": 7077888,
                        "nnz": 2263037,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 374784,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1751040,
                        "linear_total": 7077888,
                        "nnz": 1757236,
                        "total": 7086912
                    },
                    "8": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 224256,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1993728,
                        "linear_total": 7077888,
                        "nnz": 2000210,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 101376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1281024,
                        "linear_total": 7077888,
                        "nnz": 1286850,
                        "total": 7086720
                    }
                },
                "linear_nnz": 22047744,
                "linear_sparsity": 74.04152199074075,
                "linear_total": 84934656,
                "nnz": 45961684,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        10,
                        2,
                        3
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108882242,
                "total_sparsity": 57.78771344550382
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-67500": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 69.8549257759784,
                "HasAns_f1": 76.5912293488606,
                "HasAns_total": 5928,
                "NoAns_exact": 73.40622371740959,
                "NoAns_f1": 73.40622371740959,
                "NoAns_total": 5945,
                "best_exact": 71.63311715657373,
                "best_exact_thresh": 0.0,
                "best_f1": 74.9964463556006,
                "best_f1_thresh": 0.0,
                "exact": 71.63311715657373,
                "f1": 74.99644635560065,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 19.925880737304688,
                "eval_elapsed_time": 24.433438649983145
            },
            "speedup": 2.21478049486557,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 396288,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1575936,
                        "linear_total": 7077888,
                        "nnz": 1581954,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 560640,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1543680,
                        "linear_total": 7077888,
                        "nnz": 1549613,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 118272,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1101312,
                        "linear_total": 7077888,
                        "nnz": 1106957,
                        "total": 7086528
                    },
                    "11": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 152064,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1331712,
                        "linear_total": 7077888,
                        "nnz": 1337571,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 599040,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2368512,
                        "linear_total": 7077888,
                        "nnz": 2375238,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 646656,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2416128,
                        "linear_total": 7077888,
                        "nnz": 2422885,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 672768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2442240,
                        "linear_total": 7077888,
                        "nnz": 2449014,
                        "total": 7087296
                    },
                    "5": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 609792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1986048,
                        "linear_total": 7077888,
                        "nnz": 1992397,
                        "total": 7086912
                    },
                    "6": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 486912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2256384,
                        "linear_total": 7077888,
                        "nnz": 2263037,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 374784,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1751040,
                        "linear_total": 7077888,
                        "nnz": 1757236,
                        "total": 7086912
                    },
                    "8": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 224256,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1993728,
                        "linear_total": 7077888,
                        "nnz": 2000210,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 101376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1281024,
                        "linear_total": 7077888,
                        "nnz": 1286850,
                        "total": 7086720
                    }
                },
                "linear_nnz": 22047744,
                "linear_sparsity": 74.04152199074075,
                "linear_total": 84934656,
                "nnz": 45961684,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        10,
                        2,
                        3
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108882242,
                "total_sparsity": 57.78771344550382
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-80000": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 70.25978407557355,
                "HasAns_f1": 77.05720116289781,
                "HasAns_total": 5928,
                "NoAns_exact": 72.86795626576955,
                "NoAns_f1": 72.86795626576955,
                "NoAns_total": 5945,
                "best_exact": 71.56573738734944,
                "best_exact_thresh": 0.0,
                "best_f1": 74.95957959181825,
                "best_f1_thresh": 0.0,
                "exact": 71.56573738734944,
                "f1": 74.95957959181833,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 19.93493965148926,
                "eval_elapsed_time": 24.43335940502584
            },
            "speedup": 2.213774045546364,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 396288,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1575936,
                        "linear_total": 7077888,
                        "nnz": 1581954,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 560640,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1543680,
                        "linear_total": 7077888,
                        "nnz": 1549613,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 118272,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1101312,
                        "linear_total": 7077888,
                        "nnz": 1106957,
                        "total": 7086528
                    },
                    "11": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 152064,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1331712,
                        "linear_total": 7077888,
                        "nnz": 1337571,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 599040,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2368512,
                        "linear_total": 7077888,
                        "nnz": 2375238,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 646656,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2416128,
                        "linear_total": 7077888,
                        "nnz": 2422885,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 672768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2442240,
                        "linear_total": 7077888,
                        "nnz": 2449014,
                        "total": 7087296
                    },
                    "5": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 609792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1986048,
                        "linear_total": 7077888,
                        "nnz": 1992397,
                        "total": 7086912
                    },
                    "6": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 486912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2256384,
                        "linear_total": 7077888,
                        "nnz": 2263037,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 374784,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1751040,
                        "linear_total": 7077888,
                        "nnz": 1757236,
                        "total": 7086912
                    },
                    "8": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 224256,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1993728,
                        "linear_total": 7077888,
                        "nnz": 2000210,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 101376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1281024,
                        "linear_total": 7077888,
                        "nnz": 1286850,
                        "total": 7086720
                    }
                },
                "linear_nnz": 22047744,
                "linear_sparsity": 74.04152199074075,
                "linear_total": 84934656,
                "nnz": 45961684,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        10,
                        2,
                        3
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108882242,
                "total_sparsity": 57.78771344550382
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        },
        "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-82350": {
            "config": {
                "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "architectures": [
                    "BertForQuestionAnswering"
                ],
                "attention_probs_dropout_prob": 0.1,
                "gradient_checkpointing": false,
                "hidden_act": "gelu",
                "hidden_dropout_prob": 0.1,
                "hidden_size": 768,
                "initializer_range": 0.02,
                "intermediate_size": 3072,
                "layer_norm_eps": 1e-12,
                "max_position_embeddings": 512,
                "model_type": "bert",
                "num_attention_heads": 12,
                "num_hidden_layers": 12,
                "pad_token_id": 0,
                "position_embedding_type": "absolute",
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        3,
                        10,
                        2
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "transformers_version": "4.5.1",
                "type_vocab_size": 2,
                "use_cache": true,
                "vocab_size": 30522
            },
            "eval_metrics": {
                "HasAns_exact": 70.24291497975709,
                "HasAns_f1": 77.0078917303485,
                "HasAns_total": 5928,
                "NoAns_exact": 72.59882253994954,
                "NoAns_f1": 72.59882253994954,
                "NoAns_total": 5945,
                "best_exact": 71.42255537774783,
                "best_exact_thresh": 0.0,
                "best_f1": 74.80020063821324,
                "best_f1_thresh": 0.0,
                "exact": 71.42255537774783,
                "f1": 74.80020063821331,
                "total": 11873
            },
            "model_args": {
                "cache_dir": null,
                "config_name": null,
                "model_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                "tokenizer_name": null,
                "use_fast_tokenizer": true
            },
            "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682/checkpoint-155000",
            "sparse_args": {
                "ampere_pruning_method": "disabled",
                "attention_block_cols": 1,
                "attention_block_rows": 1,
                "attention_lambda": 1.0,
                "attention_output_with_dense": 0,
                "attention_pruning_method": "topK",
                "bias_mask": true,
                "dense_block_cols": 1,
                "dense_block_rows": 1,
                "dense_lambda": 1.0,
                "dense_pruning_method": "topK",
                "distil_alpha_ce": 0.1,
                "distil_alpha_teacher": 0.9,
                "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                "distil_temperature": 2.0,
                "eval_with_current_patch_params": false,
                "final_ampere_temperature": 20.0,
                "final_finetune": 1,
                "final_threshold": 0.5,
                "final_warmup": 0,
                "gelu_patch": false,
                "gelu_patch_steps": 50000,
                "initial_ampere_temperature": 0.0,
                "initial_threshold": 1.0,
                "initial_warmup": 0,
                "layer_norm_patch": false,
                "layer_norm_patch_start_delta": 0.99,
                "layer_norm_patch_steps": 50000,
                "linear_min_parameters": 0.005,
                "mask_init": "constant",
                "mask_scale": 0.0,
                "mask_scores_learning_rate": 0.01,
                "qat": false,
                "qconfig": "default",
                "regularization": "",
                "regularization_final_lambda": 0,
                "rewind_model_name_or_path": null
            },
            "speed": {
                "cuda_eval_elapsed_time": 19.934484786987305,
                "eval_elapsed_time": 24.5502042430453
            },
            "speedup": 2.213824559379023,
            "stats": {
                "layers": {
                    "0": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 396288,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1575936,
                        "linear_total": 7077888,
                        "nnz": 1581954,
                        "total": 7086720
                    },
                    "1": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 560640,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1543680,
                        "linear_total": 7077888,
                        "nnz": 1549613,
                        "total": 7086528
                    },
                    "10": {
                        "linear_attention_nnz": 983040,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 118272,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1101312,
                        "linear_total": 7077888,
                        "nnz": 1106957,
                        "total": 7086528
                    },
                    "11": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 152064,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1331712,
                        "linear_total": 7077888,
                        "nnz": 1337571,
                        "total": 7086720
                    },
                    "2": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 599040,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2368512,
                        "linear_total": 7077888,
                        "nnz": 2375238,
                        "total": 7087296
                    },
                    "3": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 646656,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2416128,
                        "linear_total": 7077888,
                        "nnz": 2422885,
                        "total": 7087296
                    },
                    "4": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 672768,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2442240,
                        "linear_total": 7077888,
                        "nnz": 2449014,
                        "total": 7087296
                    },
                    "5": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 609792,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1986048,
                        "linear_total": 7077888,
                        "nnz": 1992397,
                        "total": 7086912
                    },
                    "6": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 486912,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 2256384,
                        "linear_total": 7077888,
                        "nnz": 2263037,
                        "total": 7087296
                    },
                    "7": {
                        "linear_attention_nnz": 1376256,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 374784,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1751040,
                        "linear_total": 7077888,
                        "nnz": 1757236,
                        "total": 7086912
                    },
                    "8": {
                        "linear_attention_nnz": 1769472,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 224256,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1993728,
                        "linear_total": 7077888,
                        "nnz": 2000210,
                        "total": 7087296
                    },
                    "9": {
                        "linear_attention_nnz": 1179648,
                        "linear_attention_total": 2359296,
                        "linear_dense_nnz": 101376,
                        "linear_dense_total": 4718592,
                        "linear_nnz": 1281024,
                        "linear_total": 7077888,
                        "nnz": 1286850,
                        "total": 7086720
                    }
                },
                "linear_nnz": 22047744,
                "linear_sparsity": 74.04152199074075,
                "linear_total": 84934656,
                "nnz": 45961684,
                "pruned_heads": {
                    "0": [
                        0,
                        2,
                        4,
                        5,
                        6,
                        7
                    ],
                    "1": [
                        0,
                        2,
                        3,
                        5,
                        6,
                        7,
                        8
                    ],
                    "10": [
                        1,
                        4,
                        5,
                        6,
                        7,
                        8,
                        9
                    ],
                    "11": [
                        3,
                        5,
                        7,
                        8,
                        10,
                        11
                    ],
                    "2": [
                        8,
                        4,
                        7
                    ],
                    "3": [
                        2,
                        4,
                        6
                    ],
                    "4": [
                        8,
                        2,
                        11
                    ],
                    "5": [
                        1,
                        2,
                        6,
                        7,
                        11
                    ],
                    "6": [
                        10,
                        2,
                        3
                    ],
                    "7": [
                        1,
                        3,
                        6,
                        7,
                        11
                    ],
                    "8": [
                        0,
                        8,
                        4
                    ],
                    "9": [
                        1,
                        2,
                        4,
                        5,
                        7,
                        10
                    ]
                },
                "total": 108882242,
                "total_sparsity": 57.78771344550382
            },
            "training_args": {
                "_n_gpu": 1,
                "adafactor": false,
                "adam_beta1": 0.9,
                "adam_beta2": 0.999,
                "adam_epsilon": 1e-08,
                "dataloader_drop_last": false,
                "dataloader_num_workers": 0,
                "dataloader_pin_memory": true,
                "ddp_find_unused_parameters": null,
                "debug": false,
                "deepspeed": null,
                "disable_tqdm": false,
                "do_eval": 1,
                "do_predict": false,
                "do_train": 1,
                "eval_accumulation_steps": null,
                "eval_steps": 2500,
                "evaluation_strategy": "steps",
                "fp16": false,
                "fp16_backend": "auto",
                "fp16_full_eval": false,
                "fp16_opt_level": "O1",
                "gradient_accumulation_steps": 1,
                "greater_is_better": null,
                "group_by_length": false,
                "ignore_data_skip": false,
                "label_names": null,
                "label_smoothing_factor": 0.0,
                "learning_rate": 3e-05,
                "length_column_name": "length",
                "load_best_model_at_end": false,
                "local_rank": -1,
                "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "logging_first_step": false,
                "logging_steps": 250,
                "logging_strategy": "steps",
                "lr_scheduler_type": "linear",
                "max_grad_norm": 1.0,
                "max_steps": -1,
                "metric_for_best_model": null,
                "mp_parameters": "",
                "no_cuda": false,
                "num_train_epochs": 10,
                "optimize_model_before_eval": "disabled",
                "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "overwrite_output_dir": 1,
                "past_index": -1,
                "per_device_eval_batch_size": 128,
                "per_device_train_batch_size": 16,
                "per_gpu_eval_batch_size": null,
                "per_gpu_train_batch_size": null,
                "prediction_loss_only": false,
                "remove_unused_columns": true,
                "report_to": [
                    "tensorboard",
                    "wandb"
                ],
                "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--5f4658054e1d8682",
                "save_steps": 2500,
                "save_strategy": "steps",
                "save_total_limit": 50,
                "seed": 17,
                "sharded_ddp": [],
                "skip_memory_metrics": false,
                "tpu_metrics_debug": false,
                "tpu_num_cores": null,
                "warmup_ratio": 0.0,
                "warmup_steps": 10,
                "weight_decay": 0.0
            }
        }
    }
}