{
    "Block/struct method, bs= 32x32, v=1, s=b": [
        {
            "annotate": "34",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.4.2",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 71.2719298245614,
                    "HasAns_f1": 78.12614467791248,
                    "HasAns_total": 5928,
                    "NoAns_exact": 74.61732548359966,
                    "NoAns_f1": 74.61732548359966,
                    "NoAns_total": 5945,
                    "best_exact": 72.9470226564474,
                    "best_exact_thresh": 0.0,
                    "best_f1": 76.36922308183821,
                    "best_f1_thresh": 0.0,
                    "exact": 72.9470226564474,
                    "f1": 76.36922308183819,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 5,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5.0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 26.076661651611328,
                    "eval_elapsed_time": 30.333236434031278
                },
                "speedup": 1.6923735173130998,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1624064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1434624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3058688,
                            "linear_total": 7077888,
                            "nnz": 3066150,
                            "total": 7087680
                        },
                        "1": {
                            "linear_attention_nnz": 1034240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1509888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2544128,
                            "linear_total": 7077888,
                            "nnz": 2551127,
                            "total": 7087104
                        },
                        "10": {
                            "linear_attention_nnz": 760832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 251904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1012736,
                            "linear_total": 7077888,
                            "nnz": 1018692,
                            "total": 7087104
                        },
                        "11": {
                            "linear_attention_nnz": 533504,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 417792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 951296,
                            "linear_total": 7077888,
                            "nnz": 957200,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1571840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1566720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3138560,
                            "linear_total": 7077888,
                            "nnz": 3145948,
                            "total": 7087488
                        },
                        "3": {
                            "linear_attention_nnz": 1561600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1681920,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3243520,
                            "linear_total": 7077888,
                            "nnz": 3250951,
                            "total": 7087296
                        },
                        "4": {
                            "linear_attention_nnz": 1954816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1637376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3592192,
                            "linear_total": 7077888,
                            "nnz": 3600170,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1661952,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1532928,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3194880,
                            "linear_total": 7077888,
                            "nnz": 3202406,
                            "total": 7087488
                        },
                        "6": {
                            "linear_attention_nnz": 1730560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1167360,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2897920,
                            "linear_total": 7077888,
                            "nnz": 2905400,
                            "total": 7087680
                        },
                        "7": {
                            "linear_attention_nnz": 1313792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 892416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2206208,
                            "linear_total": 7077888,
                            "nnz": 2212933,
                            "total": 7087104
                        },
                        "8": {
                            "linear_attention_nnz": 1441792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 473088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1914880,
                            "linear_total": 7077888,
                            "nnz": 1921620,
                            "total": 7087488
                        },
                        "9": {
                            "linear_attention_nnz": 906240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 222720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1128960,
                            "linear_total": 7077888,
                            "nnz": 1134961,
                            "total": 7086912
                        }
                    },
                    "linear_nnz": 28883968,
                    "linear_sparsity": 65.99271797839505,
                    "linear_total": 84934656,
                    "nnz": 52806280,
                    "pruned_heads": {
                        "0": [
                            4
                        ],
                        "1": [
                            0,
                            8,
                            2,
                            5
                        ],
                        "10": [
                            1,
                            4,
                            5,
                            6
                        ],
                        "11": [
                            3,
                            11,
                            5,
                            7
                        ],
                        "2": [
                            8,
                            4
                        ],
                        "3": [
                            2,
                            4,
                            6
                        ],
                        "4": [],
                        "5": [
                            1,
                            2
                        ],
                        "6": [
                            3
                        ],
                        "7": [
                            3,
                            11,
                            6,
                            7
                        ],
                        "8": [
                            0,
                            8
                        ],
                        "9": [
                            1,
                            4,
                            5,
                            7,
                            10
                        ]
                    },
                    "total": 108887042,
                    "total_sparsity": 51.50361417660698
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 20,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 5400,
                    "weight_decay": 0.0
                }
            },
            "f1": 76.36922308183819,
            "fill_rate": 0.34007282021604945,
            "speedup": 1.6923735173130998
        }
    ],
    "Block/struct method, final fine tuned, s=b": [
        {
            "annotate": "100",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 73.31309041835358,
                    "HasAns_f1": 79.77022173419074,
                    "HasAns_total": 5928,
                    "NoAns_exact": 74.80235492010092,
                    "NoAns_f1": 74.80235492010092,
                    "NoAns_total": 5945,
                    "best_exact": 74.05878884864819,
                    "best_exact_thresh": 0.0,
                    "best_f1": 77.2827317813764,
                    "best_f1_thresh": 0.0,
                    "exact": 74.05878884864819,
                    "f1": 77.28273178137641,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-70000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 44.72616534423828,
                    "eval_elapsed_time": 49.33306856802665
                },
                "speedup": 0.9867032252700421,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 84934656,
                    "linear_sparsity": 0.0,
                    "linear_total": 84934656,
                    "nnz": 108893186,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [],
                        "11": [],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": []
                    },
                    "total": 108893186,
                    "total_sparsity": 0.0
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 77.28273178137641,
            "fill_rate": 1.0,
            "speedup": 0.9867032252700421
        },
        {
            "annotate": "100",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 73.59986504723346,
                    "HasAns_f1": 80.16436934694633,
                    "HasAns_total": 5928,
                    "NoAns_exact": 74.02859545836837,
                    "NoAns_f1": 74.02859545836837,
                    "NoAns_total": 5945,
                    "best_exact": 73.81453718521014,
                    "best_exact_thresh": 0.0,
                    "best_f1": 77.09208974047822,
                    "best_f1_thresh": 0.0,
                    "exact": 73.81453718521014,
                    "f1": 77.09208974047823,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-75000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 44.508157470703125,
                    "eval_elapsed_time": 49.17035996296909
                },
                "speedup": 0.9915362510382508,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 84934656,
                    "linear_sparsity": 0.0,
                    "linear_total": 84934656,
                    "nnz": 108893186,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [],
                        "11": [],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": []
                    },
                    "total": 108893186,
                    "total_sparsity": 0.0
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 77.09208974047823,
            "fill_rate": 1.0,
            "speedup": 0.9915362510382508
        },
        {
            "annotate": "100",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForQuestionAnswering"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "transformers_version": "4.5.1",
                    "type_vocab_size": 2,
                    "use_cache": true,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "HasAns_exact": 73.88663967611336,
                    "HasAns_f1": 80.41695213427651,
                    "HasAns_total": 5928,
                    "NoAns_exact": 73.77628259041211,
                    "NoAns_f1": 73.77628259041211,
                    "NoAns_total": 5945,
                    "best_exact": 73.8313821275162,
                    "best_exact_thresh": 0.0,
                    "best_f1": 77.09186324029237,
                    "best_f1_thresh": 0.0,
                    "exact": 73.8313821275162,
                    "f1": 77.09186324029237,
                    "total": 11873
                },
                "path": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-82350",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/squadv2_test_0/hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "twmkn9/bert-base-uncased-squad2",
                    "distil_temperature": 2.0,
                    "eval_with_current_patch_params": false,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "gelu_patch": false,
                    "gelu_patch_steps": 50000,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "layer_norm_patch": false,
                    "layer_norm_patch_start_delta": 0.99,
                    "layer_norm_patch_steps": 50000,
                    "linear_min_parameters": 0.005,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "qat": false,
                    "qconfig": "default",
                    "regularization": "",
                    "regularization_final_lambda": 0,
                    "rewind_model_name_or_path": null
                },
                "speed": {
                    "cuda_eval_elapsed_time": 44.50070120239258,
                    "eval_elapsed_time": 49.16471692698542
                },
                "speedup": 0.9917023868547125,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 2359296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 4718592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 7077888,
                            "linear_total": 7077888,
                            "nnz": 7087872,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 84934656,
                    "linear_sparsity": 0.0,
                    "linear_total": 84934656,
                    "nnz": 108893186,
                    "pruned_heads": {
                        "0": [],
                        "1": [],
                        "10": [],
                        "11": [],
                        "2": [],
                        "3": [],
                        "4": [],
                        "5": [],
                        "6": [],
                        "7": [],
                        "8": [],
                        "9": []
                    },
                    "total": 108893186,
                    "total_sparsity": 0.0
                },
                "training_args": {
                    "_n_gpu": 1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 2500,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "length_column_name": "length",
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "mp_parameters": "",
                    "no_cuda": false,
                    "num_train_epochs": 10,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 16,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": [
                        "tensorboard",
                        "wandb"
                    ],
                    "run_name": "/data_2to/devel_data/nn_pruning/output/squad_v2_test_final_fine_tune/fine_tuned_hp_dn-squad_v2_v2wn1_od-__data_2to__devel_data__nn_pruning__output__squadv2_test_0_es-steps_pdebs128_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squadv2_--3c220d21d68ec2de",
                    "save_steps": 2500,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": [],
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "f1": 77.09186324029237,
            "fill_rate": 1.0,
            "speedup": 0.9917023868547125
        }
    ]
}