{
    "Block/struct method, bs= 32x32, v=1, s=b": [
        {
            "annotate": "19",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8213958227203261,
                    "eval_loss": 0.6194782853126526
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8245524816924329,
                    "eval_loss": 0.5971441864967346
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4/checkpoint-115000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.968112411499025,
                    "eval_elapsed_time": 22.53923809621483
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 21.05781527709961,
                    "eval_elapsed_time": 22.641935867257416
                },
                "speedup": 2.3413865199705843,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 710656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 513024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1223680,
                            "linear_total": 7077888,
                            "nnz": 1229582,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 689152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 595968,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1285120,
                            "linear_total": 7077888,
                            "nnz": 1290948,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 280576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 110592,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 391168,
                            "linear_total": 7077888,
                            "nnz": 396200,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 198656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 33792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 232448,
                            "linear_total": 7077888,
                            "nnz": 237910,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 966656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 830976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1797632,
                            "linear_total": 7077888,
                            "nnz": 1803965,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 939008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1022976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1961984,
                            "linear_total": 7077888,
                            "nnz": 1968442,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1017856,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1059840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2077696,
                            "linear_total": 7077888,
                            "nnz": 2084434,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1278976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 861696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2140672,
                            "linear_total": 7077888,
                            "nnz": 2147537,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 845824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1644544,
                            "linear_total": 7077888,
                            "nnz": 1650888,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 886784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 700416,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1587200,
                            "linear_total": 7077888,
                            "nnz": 1593448,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 629760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 485376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1115136,
                            "linear_total": 7077888,
                            "nnz": 1121084,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 526336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 185856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 712192,
                            "linear_total": 7077888,
                            "nnz": 717689,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 16169472,
                    "linear_sparsity": 80.96245659722221,
                    "linear_total": 84934656,
                    "nnz": 40672210,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            8,
                            1,
                            2,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 62.85118666107281
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.1903754340277779,
            "matched": 82.13958227203261,
            "mismatched": 82.45524816924329,
            "speedup": 2.3413865199705843
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8223127865511971,
                    "eval_loss": 0.6188907027244568
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8256712774613507,
                    "eval_loss": 0.5843316316604614
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4/checkpoint-140000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.611759689331056,
                    "eval_elapsed_time": 22.18333603721112
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 20.717923553466797,
                    "eval_elapsed_time": 22.301769183017313
                },
                "speedup": 2.381866298146476,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 662528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 474624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1137152,
                            "linear_total": 7077888,
                            "nnz": 1142965,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 675840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 568320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1244160,
                            "linear_total": 7077888,
                            "nnz": 1249970,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 252928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 109056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 361984,
                            "linear_total": 7077888,
                            "nnz": 366983,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 182272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 33792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 216064,
                            "linear_total": 7077888,
                            "nnz": 221430,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 961536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1760256,
                            "linear_total": 7077888,
                            "nnz": 1766568,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 894976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 996864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1891840,
                            "linear_total": 7077888,
                            "nnz": 1898281,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 845824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1010688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1856512,
                            "linear_total": 7077888,
                            "nnz": 1862866,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1148928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 838656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1987584,
                            "linear_total": 7077888,
                            "nnz": 1994370,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 801792,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 771072,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1572864,
                            "linear_total": 7077888,
                            "nnz": 1579094,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 820224,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 680448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1500672,
                            "linear_total": 7077888,
                            "nnz": 1506875,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 567296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 477696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1044992,
                            "linear_total": 7077888,
                            "nnz": 1050935,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 502784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 677888,
                            "linear_total": 7077888,
                            "nnz": 683346,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15251968,
                    "linear_sparsity": 82.04270351080247,
                    "linear_total": 84934656,
                    "nnz": 39753766,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 63.69006669041614
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.17957296489197527,
            "matched": 82.23127865511971,
            "mismatched": 82.56712774613507,
            "speedup": 2.381866298146476
        },
        {
            "annotate": "17",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8227203260315843,
                    "eval_loss": 0.6198351383209229
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.82740032546786,
                    "eval_loss": 0.5863709449768066
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4/checkpoint-145000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 10
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.451122985839845,
                    "eval_elapsed_time": 22.043534694239497
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 20.540529357910156,
                    "eval_elapsed_time": 22.19288508500904
                },
                "speedup": 2.4005750580789247,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 685056,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 474624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1159680,
                            "linear_total": 7077888,
                            "nnz": 1165493,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 683008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 568320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1251328,
                            "linear_total": 7077888,
                            "nnz": 1257138,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 254976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 109056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 364032,
                            "linear_total": 7077888,
                            "nnz": 368999,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 188416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 33792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 222208,
                            "linear_total": 7077888,
                            "nnz": 227638,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 931840,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1730560,
                            "linear_total": 7077888,
                            "nnz": 1736872,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 891904,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 996864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1888768,
                            "linear_total": 7077888,
                            "nnz": 1895209,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 847872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1010688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1858560,
                            "linear_total": 7077888,
                            "nnz": 1864914,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1118208,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 838656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1956864,
                            "linear_total": 7077888,
                            "nnz": 1963554,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 805888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 769536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1575424,
                            "linear_total": 7077888,
                            "nnz": 1581653,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 816128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 680448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1496576,
                            "linear_total": 7077888,
                            "nnz": 1502779,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 562176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 477696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1039872,
                            "linear_total": 7077888,
                            "nnz": 1045815,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 506880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 681984,
                            "linear_total": 7077888,
                            "nnz": 687442,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 15225856,
                    "linear_sparsity": 82.07344714506173,
                    "linear_total": 84934656,
                    "nnz": 39727589,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 63.713976000649666
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.1792655285493826,
            "matched": 82.27203260315844,
            "mismatched": 82.740032546786,
            "speedup": 2.4005750580789247
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8167091186958737,
                    "eval_loss": 0.6105015277862549
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8164157851912124,
                    "eval_loss": 0.5977670550346375
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.94356623840332,
                    "eval_elapsed_time": 18.50567529629916
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 16.96883401489258,
                    "eval_elapsed_time": 18.56157809589058
                },
                "speedup": 2.8975278910432105,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 555008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 268800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 823808,
                            "linear_total": 7077888,
                            "nnz": 829295,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 592896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 897024,
                            "linear_total": 7077888,
                            "nnz": 902566,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 230400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 70656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 301056,
                            "linear_total": 7077888,
                            "nnz": 306030,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 133120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 156160,
                            "linear_total": 7077888,
                            "nnz": 161199,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 582656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 459264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1041920,
                            "linear_total": 7077888,
                            "nnz": 1047499,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 742400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1356800,
                            "linear_total": 7077888,
                            "nnz": 1362736,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 730112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 602112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1332224,
                            "linear_total": 7077888,
                            "nnz": 1338120,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 840704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 489984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1330688,
                            "linear_total": 7077888,
                            "nnz": 1336799,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 523264,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 959488,
                            "linear_total": 7077888,
                            "nnz": 965116,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 568320,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 453120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1021440,
                            "linear_total": 7077888,
                            "nnz": 1027079,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 483328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 327168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 810496,
                            "linear_total": 7077888,
                            "nnz": 816085,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 376832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 112128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 488960,
                            "linear_total": 7077888,
                            "nnz": 494185,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 10520064,
                    "linear_sparsity": 87.61393229166666,
                    "linear_total": 84934656,
                    "nnz": 35016792,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 68.01668092941007
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.12386067708333348,
            "matched": 81.67091186958737,
            "mismatched": 81.64157851912124,
            "speedup": 2.8975278910432105
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8137544574630667,
                    "eval_loss": 0.6326338052749634
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8127542717656632,
                    "eval_loss": 0.6192973852157593
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20/checkpoint-130000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.52580699157715,
                    "eval_elapsed_time": 18.086638130247593
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 16.60585534667969,
                    "eval_elapsed_time": 18.22978367190808
                },
                "speedup": 2.9707750898055454,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 495616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 224256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 719872,
                            "linear_total": 7077888,
                            "nnz": 725266,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 540672,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 258048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 798720,
                            "linear_total": 7077888,
                            "nnz": 804168,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 223232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 64512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 287744,
                            "linear_total": 7077888,
                            "nnz": 292714,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 118784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 141824,
                            "linear_total": 7077888,
                            "nnz": 146831,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 570368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 413184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 983552,
                            "linear_total": 7077888,
                            "nnz": 989101,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 688128,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 571392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1259520,
                            "linear_total": 7077888,
                            "nnz": 1265364,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 663552,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 556032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1219584,
                            "linear_total": 7077888,
                            "nnz": 1225418,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 753664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 450048,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1203712,
                            "linear_total": 7077888,
                            "nnz": 1209765,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 525312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 405504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 930816,
                            "linear_total": 7077888,
                            "nnz": 936424,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 524288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 411648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 935936,
                            "linear_total": 7077888,
                            "nnz": 941548,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 355328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 307200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 662528,
                            "linear_total": 7077888,
                            "nnz": 667976,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 385024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 102912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 487936,
                            "linear_total": 7077888,
                            "nnz": 493059,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 9631744,
                    "linear_sparsity": 88.6598186728395,
                    "linear_total": 84934656,
                    "nnz": 34127717,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 68.8287361685846
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.11340181327160492,
            "matched": 81.37544574630667,
            "mismatched": 81.27542717656631,
            "speedup": 2.9707750898055454
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8129393785022924,
                    "eval_loss": 0.6277854442596436
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8157038242473555,
                    "eval_loss": 0.6225855350494385
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20/checkpoint-140000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.467503906250002,
                    "eval_elapsed_time": 18.033160428516567
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 16.515043060302734,
                    "eval_elapsed_time": 18.09035024512559
                },
                "speedup": 2.9812930987603186,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 482304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 221184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 703488,
                            "linear_total": 7077888,
                            "nnz": 708848,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 527360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 256512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 783872,
                            "linear_total": 7077888,
                            "nnz": 789319,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 214016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 64512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 278528,
                            "linear_total": 7077888,
                            "nnz": 283530,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 117760,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 140800,
                            "linear_total": 7077888,
                            "nnz": 145807,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 563200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 411648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 974848,
                            "linear_total": 7077888,
                            "nnz": 980396,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 677888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 569856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1247744,
                            "linear_total": 7077888,
                            "nnz": 1253587,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 664576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1217536,
                            "linear_total": 7077888,
                            "nnz": 1223368,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 712704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 445440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1158144,
                            "linear_total": 7077888,
                            "nnz": 1164194,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 510976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 405504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 916480,
                            "linear_total": 7077888,
                            "nnz": 922088,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 504832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 410112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 914944,
                            "linear_total": 7077888,
                            "nnz": 920555,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 344064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 648192,
                            "linear_total": 7077888,
                            "nnz": 653638,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 374784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 102912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 477696,
                            "linear_total": 7077888,
                            "nnz": 482851,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 9462272,
                    "linear_sparsity": 88.85935088734568,
                    "linear_total": 84934656,
                    "nnz": 33958264,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 68.98350960889485
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.1114064911265431,
            "matched": 81.29393785022924,
            "mismatched": 81.57038242473556,
            "speedup": 2.9812930987603186
        },
        {
            "annotate": "11",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8107997962302598,
                    "eval_loss": 0.6324005126953125
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8147884458909682,
                    "eval_loss": 0.6208154559135437
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20/checkpoint-145000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 20
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.484155334472657,
                    "eval_elapsed_time": 18.08931072242558
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 16.524008483886718,
                    "eval_elapsed_time": 18.107759995386004
                },
                "speedup": 2.9782815530039586,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 479232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 221184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 700416,
                            "linear_total": 7077888,
                            "nnz": 705776,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 503808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 758784,
                            "linear_total": 7077888,
                            "nnz": 764230,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 200704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 64512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 265216,
                            "linear_total": 7077888,
                            "nnz": 270218,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 119808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 142848,
                            "linear_total": 7077888,
                            "nnz": 147855,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 562176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 411648,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 973824,
                            "linear_total": 7077888,
                            "nnz": 979372,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 679936,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 569856,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1249792,
                            "linear_total": 7077888,
                            "nnz": 1255635,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 664576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 552960,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1217536,
                            "linear_total": 7077888,
                            "nnz": 1223368,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 701440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 445440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1146880,
                            "linear_total": 7077888,
                            "nnz": 1152866,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 531456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 405504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 936960,
                            "linear_total": 7077888,
                            "nnz": 942568,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 508928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 410112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 919040,
                            "linear_total": 7077888,
                            "nnz": 924651,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 326656,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 630784,
                            "linear_total": 7077888,
                            "nnz": 636198,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 367616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 102912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 470528,
                            "linear_total": 7077888,
                            "nnz": 475651,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 9412608,
                    "linear_sparsity": 88.91782407407408,
                    "linear_total": 84934656,
                    "nnz": 33908471,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 69.02898908646897
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.11082175925925919,
            "matched": 81.07997962302598,
            "mismatched": 81.47884458909682,
            "speedup": 2.9782815530039586
        },
        {
            "annotate": "8",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8058074375955171,
                    "eval_loss": 0.6541752219200134
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.807160292921074,
                    "eval_loss": 0.6403667330741882
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl30/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.742925094604493,
                    "eval_elapsed_time": 16.3213126398623
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 14.796650604248047,
                    "eval_elapsed_time": 16.36922346521169
                },
                "speedup": 3.3300349445225725,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 428032,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 139776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 567808,
                            "linear_total": 7077888,
                            "nnz": 573051,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 418816,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 176640,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 595456,
                            "linear_total": 7077888,
                            "nnz": 600723,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 165888,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 58368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 224256,
                            "linear_total": 7077888,
                            "nnz": 228966,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 99328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 122368,
                            "linear_total": 7077888,
                            "nnz": 127183,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 532480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 802816,
                            "linear_total": 7077888,
                            "nnz": 808272,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 450560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 397824,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 848384,
                            "linear_total": 7077888,
                            "nnz": 853891,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 565248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 351744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 916992,
                            "linear_total": 7077888,
                            "nnz": 922597,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 437248,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 299520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 736768,
                            "linear_total": 7077888,
                            "nnz": 742211,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 317440,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 287232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 604672,
                            "linear_total": 7077888,
                            "nnz": 609883,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 346112,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 317952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 664064,
                            "linear_total": 7077888,
                            "nnz": 669391,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 238592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 230400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 468992,
                            "linear_total": 7077888,
                            "nnz": 474102,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 340992,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 86016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 427008,
                            "linear_total": 7077888,
                            "nnz": 432088,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 6979584,
                    "linear_sparsity": 91.7824074074074,
                    "linear_total": 84934656,
                    "nnz": 31472441,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 71.25398801713999
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.08217592592592593,
            "matched": 80.58074375955171,
            "mismatched": 80.71602929210741,
            "speedup": 3.3300349445225725
        },
        {
            "annotate": "8",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8037697401935813,
                    "eval_loss": 0.6670960783958435
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8112286411716843,
                    "eval_loss": 0.6344207525253296
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl30/checkpoint-140000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.763623336791992,
                    "eval_elapsed_time": 16.32821988593787
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 14.803303848266602,
                    "eval_elapsed_time": 16.384684168733656
                },
                "speedup": 3.3253663162189233,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 431104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 139776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 570880,
                            "linear_total": 7077888,
                            "nnz": 576123,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 397312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 572416,
                            "linear_total": 7077888,
                            "nnz": 577682,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 160768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 58368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 219136,
                            "linear_total": 7077888,
                            "nnz": 223878,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 98304,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 121344,
                            "linear_total": 7077888,
                            "nnz": 126159,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 536576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 806912,
                            "linear_total": 7077888,
                            "nnz": 812368,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 448512,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 396288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 844800,
                            "linear_total": 7077888,
                            "nnz": 850306,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 550912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 351744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 902656,
                            "linear_total": 7077888,
                            "nnz": 908261,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 452608,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 299520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 752128,
                            "linear_total": 7077888,
                            "nnz": 757571,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 297984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 287232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 585216,
                            "linear_total": 7077888,
                            "nnz": 590427,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 343040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 317952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 660992,
                            "linear_total": 7077888,
                            "nnz": 666319,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 229376,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 230400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 459776,
                            "linear_total": 7077888,
                            "nnz": 464886,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 332800,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 86016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 418816,
                            "linear_total": 7077888,
                            "nnz": 423928,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 6915072,
                    "linear_sparsity": 91.85836226851852,
                    "linear_total": 84934656,
                    "nnz": 31407991,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 71.31285477209856
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.08141637731481477,
            "matched": 80.37697401935813,
            "mismatched": 81.12286411716842,
            "speedup": 3.3253663162189233
        },
        {
            "annotate": "8",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8018339276617422,
                    "eval_loss": 0.6773494482040405
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.807567127746135,
                    "eval_loss": 0.6500887870788574
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl30/checkpoint-145000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.748979888916017,
                    "eval_elapsed_time": 16.32735802885145
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 14.786631912231446,
                    "eval_elapsed_time": 16.38811536040157
                },
                "speedup": 3.3286678888488157,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 449536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 139776,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 589312,
                            "linear_total": 7077888,
                            "nnz": 594555,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 403456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 172032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 575488,
                            "linear_total": 7077888,
                            "nnz": 580720,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 171008,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 58368,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 229376,
                            "linear_total": 7077888,
                            "nnz": 234118,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 100352,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 123392,
                            "linear_total": 7077888,
                            "nnz": 128207,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 536576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 270336,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 806912,
                            "linear_total": 7077888,
                            "nnz": 812368,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 450560,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 394752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 845312,
                            "linear_total": 7077888,
                            "nnz": 850817,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 538624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 351744,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 890368,
                            "linear_total": 7077888,
                            "nnz": 895973,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 434176,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 299520,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 733696,
                            "linear_total": 7077888,
                            "nnz": 739139,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 305152,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 287232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 592384,
                            "linear_total": 7077888,
                            "nnz": 597595,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 328704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 317952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 646656,
                            "linear_total": 7077888,
                            "nnz": 651983,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 223232,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 230400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 453632,
                            "linear_total": 7077888,
                            "nnz": 458710,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 335872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 86016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 421888,
                            "linear_total": 7077888,
                            "nnz": 426936,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 6908416,
                    "linear_sparsity": 91.86619888117285,
                    "linear_total": 84934656,
                    "nnz": 31401204,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 71.31905382044464
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.08133801118827155,
            "matched": 80.18339276617422,
            "mismatched": 80.7567127746135,
            "speedup": 3.3286678888488157
        },
        {
            "annotate": "6",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8053998981151299,
                    "eval_loss": 0.6555120944976807
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8049227013832384,
                    "eval_loss": 0.6468992829322815
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl40/checkpoint-140000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.990292449951172,
                    "eval_elapsed_time": 15.553386124782264
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 14.01429052734375,
                    "eval_elapsed_time": 15.591836652718484
                },
                "speedup": 3.509180092206226,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 362496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 93696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 456192,
                            "linear_total": 7077888,
                            "nnz": 461309,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 310272,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 98304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 408576,
                            "linear_total": 7077888,
                            "nnz": 413600,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 116736,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 46080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 162816,
                            "linear_total": 7077888,
                            "nnz": 167390,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 86016,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 109056,
                            "linear_total": 7077888,
                            "nnz": 113871,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 515072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 192000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 707072,
                            "linear_total": 7077888,
                            "nnz": 712413,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 431104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 285696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 716800,
                            "linear_total": 7077888,
                            "nnz": 722202,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 323584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 603136,
                            "linear_total": 7077888,
                            "nnz": 608406,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 361472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 225792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 587264,
                            "linear_total": 7077888,
                            "nnz": 592627,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 272384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 213504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 485888,
                            "linear_total": 7077888,
                            "nnz": 491019,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 269312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 524288,
                            "linear_total": 7077888,
                            "nnz": 529510,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 183296,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 382976,
                            "linear_total": 7077888,
                            "nnz": 387970,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 272384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 69120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 341504,
                            "linear_total": 7077888,
                            "nnz": 346445,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 5485568,
                    "linear_sparsity": 93.54142554012346,
                    "linear_total": 84934656,
                    "nnz": 29976845,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 72.62002189222191
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.06458574459876543,
            "matched": 80.539989811513,
            "mismatched": 80.49227013832385,
            "speedup": 3.509180092206226
        },
        {
            "annotate": "6",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.7997962302598064,
                    "eval_loss": 0.6668020486831665
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8050244100895037,
                    "eval_loss": 0.6475551724433899
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl40/checkpoint-145000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 40
                },
                "speed": {
                    "cuda_eval_elapsed_time": 13.992682876586914,
                    "eval_elapsed_time": 15.55313719343394
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 14.010611160278321,
                    "eval_elapsed_time": 15.598591247573495
                },
                "speedup": 3.5085806047715424,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 362496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 93696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 456192,
                            "linear_total": 7077888,
                            "nnz": 461309,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 294912,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 98304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 393216,
                            "linear_total": 7077888,
                            "nnz": 398240,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 113664,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 46080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 159744,
                            "linear_total": 7077888,
                            "nnz": 164286,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 81920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 104960,
                            "linear_total": 7077888,
                            "nnz": 109743,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 503808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 192000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 695808,
                            "linear_total": 7077888,
                            "nnz": 701149,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 430080,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 285696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 715776,
                            "linear_total": 7077888,
                            "nnz": 721178,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 321536,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 601088,
                            "linear_total": 7077888,
                            "nnz": 606294,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 374784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 225792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 600576,
                            "linear_total": 7077888,
                            "nnz": 605939,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 262144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 213504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 475648,
                            "linear_total": 7077888,
                            "nnz": 480779,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 266240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 521216,
                            "linear_total": 7077888,
                            "nnz": 526406,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 186368,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 386048,
                            "linear_total": 7077888,
                            "nnz": 391042,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 283648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 69120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 352768,
                            "linear_total": 7077888,
                            "nnz": 357677,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 5463040,
                    "linear_sparsity": 93.56794945987654,
                    "linear_total": 84934656,
                    "nnz": 29954125,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 72.64077367922982
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.06432050540123457,
            "matched": 79.97962302598064,
            "mismatched": 80.50244100895037,
            "speedup": 3.5085806047715424
        },
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8319918492103923,
                    "eval_loss": 0.5823615193367004
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8361472742066721,
                    "eval_loss": 0.5584035515785217
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.612513916015626,
                    "eval_elapsed_time": 26.227325464598835
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 24.69035372924805,
                    "eval_elapsed_time": 26.292322852648795
                },
                "speedup": 1.9946948904542998,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 877568,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 970752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1848320,
                            "linear_total": 7077888,
                            "nnz": 1854648,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 743424,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1047552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1790976,
                            "linear_total": 7077888,
                            "nnz": 1797162,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 352256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 518144,
                            "linear_total": 7077888,
                            "nnz": 523372,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 250880,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 56832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 307712,
                            "linear_total": 7077888,
                            "nnz": 313253,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1091584,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1325568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2417152,
                            "linear_total": 7077888,
                            "nnz": 2423935,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1209344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1611264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2820608,
                            "linear_total": 7077888,
                            "nnz": 2827769,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1282048,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1666560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2948608,
                            "linear_total": 7077888,
                            "nnz": 2955965,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1424384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1433088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2857472,
                            "linear_total": 7077888,
                            "nnz": 2864741,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1209344,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2490368,
                            "linear_total": 7077888,
                            "nnz": 2497378,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 1010688,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1118208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2128896,
                            "linear_total": 7077888,
                            "nnz": 2135480,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 947200,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 800256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1747456,
                            "linear_total": 7077888,
                            "nnz": 1754025,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 655360,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 934912,
                            "linear_total": 7077888,
                            "nnz": 940598,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 22810624,
                    "linear_sparsity": 73.14332561728395,
                    "linear_total": 84934656,
                    "nnz": 47318409,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 56.78074185208987
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.2685667438271605,
            "matched": 83.19918492103923,
            "mismatched": 83.61472742066721,
            "speedup": 1.9946948904542998
        },
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8313805399898115,
                    "eval_loss": 0.5729787349700928
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8360455655004069,
                    "eval_loss": 0.5526318550109863
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-140000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.52424349975586,
                    "eval_elapsed_time": 26.1085445843637
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 24.615995880126952,
                    "eval_elapsed_time": 26.190075170248747
                },
                "speedup": 2.0018744207135466,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 848896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 969216,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1818112,
                            "linear_total": 7077888,
                            "nnz": 1824439,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 713728,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1046016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1759744,
                            "linear_total": 7077888,
                            "nnz": 1765929,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 345088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 510976,
                            "linear_total": 7077888,
                            "nnz": 516172,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 256000,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 56832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 312832,
                            "linear_total": 7077888,
                            "nnz": 318373,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1096704,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1324032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2420736,
                            "linear_total": 7077888,
                            "nnz": 2427518,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1229824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1609728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2839552,
                            "linear_total": 7077888,
                            "nnz": 2846712,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1285120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1666560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2951680,
                            "linear_total": 7077888,
                            "nnz": 2959037,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1414144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1430016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2844160,
                            "linear_total": 7077888,
                            "nnz": 2851427,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1159168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2440192,
                            "linear_total": 7077888,
                            "nnz": 2447202,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 993280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1112064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2105344,
                            "linear_total": 7077888,
                            "nnz": 2111924,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 915456,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1714176,
                            "linear_total": 7077888,
                            "nnz": 1720744,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 636928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 916480,
                            "linear_total": 7077888,
                            "nnz": 922166,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 22633984,
                    "linear_sparsity": 73.35129726080247,
                    "linear_total": 84934656,
                    "nnz": 47141726,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 56.94211896405801
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.2664870273919753,
            "matched": 83.13805399898115,
            "mismatched": 83.60455655004068,
            "speedup": 2.0018744207135466
        },
        {
            "annotate": "26",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8303616912888436,
                    "eval_loss": 0.5739728212356567
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8365541090317331,
                    "eval_loss": 0.5615866780281067
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-145000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 5
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.514598022460937,
                    "eval_elapsed_time": 26.127468508668244
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 24.603808868408205,
                    "eval_elapsed_time": 26.16779050324112
                },
                "speedup": 2.002662075247167,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 867328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 967680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1835008,
                            "linear_total": 7077888,
                            "nnz": 1841334,
                            "total": 7087872
                        },
                        "1": {
                            "linear_attention_nnz": 719872,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1046016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1765888,
                            "linear_total": 7077888,
                            "nnz": 1772073,
                            "total": 7087872
                        },
                        "10": {
                            "linear_attention_nnz": 343040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 508928,
                            "linear_total": 7077888,
                            "nnz": 514124,
                            "total": 7087872
                        },
                        "11": {
                            "linear_attention_nnz": 252928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 56832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 309760,
                            "linear_total": 7077888,
                            "nnz": 315301,
                            "total": 7087872
                        },
                        "2": {
                            "linear_attention_nnz": 1071104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1324032,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2395136,
                            "linear_total": 7077888,
                            "nnz": 2401918,
                            "total": 7087872
                        },
                        "3": {
                            "linear_attention_nnz": 1240064,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1609728,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2849792,
                            "linear_total": 7077888,
                            "nnz": 2856952,
                            "total": 7087872
                        },
                        "4": {
                            "linear_attention_nnz": 1287168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1666560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2953728,
                            "linear_total": 7077888,
                            "nnz": 2961085,
                            "total": 7087872
                        },
                        "5": {
                            "linear_attention_nnz": 1409024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1428480,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2837504,
                            "linear_total": 7077888,
                            "nnz": 2844770,
                            "total": 7087872
                        },
                        "6": {
                            "linear_attention_nnz": 1172480,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2453504,
                            "linear_total": 7077888,
                            "nnz": 2460482,
                            "total": 7087872
                        },
                        "7": {
                            "linear_attention_nnz": 995328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1112064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2107392,
                            "linear_total": 7077888,
                            "nnz": 2113940,
                            "total": 7087872
                        },
                        "8": {
                            "linear_attention_nnz": 897024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1695744,
                            "linear_total": 7077888,
                            "nnz": 1702280,
                            "total": 7087872
                        },
                        "9": {
                            "linear_attention_nnz": 650240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 929792,
                            "linear_total": 7077888,
                            "nnz": 935478,
                            "total": 7087872
                        }
                    },
                    "linear_nnz": 22642176,
                    "linear_sparsity": 73.34165219907408,
                    "linear_total": 84934656,
                    "nnz": 47149820,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109484547,
                    "total_sparsity": 56.934726139936444
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.2665834780092592,
            "matched": 83.03616912888437,
            "mismatched": 83.6554109031733,
            "speedup": 2.002662075247167
        },
        {
            "annotate": "10",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8034640855832909,
                    "eval_loss": 0.5924767851829529
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8030919446704637,
                    "eval_loss": 0.5742725729942322
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte6_ws12000_rn-output__mnli_test2___fw4_rfl30/checkpoint-50000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 3.3547179946899415,
                    "eval_elapsed_time": 4.847926817834377
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 2.9051672096252443,
                    "eval_elapsed_time": 4.3689205115661025
                },
                "speedup": 14.634450891914463,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 316416,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 334848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 651264,
                            "linear_total": 7077888,
                            "nnz": 656474,
                            "total": 7085952
                        },
                        "1": {
                            "linear_attention_nnz": 517120,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 340992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 858112,
                            "linear_total": 7077888,
                            "nnz": 863582,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 175104,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 61440,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 236544,
                            "linear_total": 7077888,
                            "nnz": 241384,
                            "total": 7086144
                        },
                        "11": {
                            "linear_attention_nnz": 119808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 142848,
                            "linear_total": 7077888,
                            "nnz": 147759,
                            "total": 7086336
                        },
                        "2": {
                            "linear_attention_nnz": 528384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 420864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 949248,
                            "linear_total": 7077888,
                            "nnz": 954834,
                            "total": 7086336
                        },
                        "3": {
                            "linear_attention_nnz": 415744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 571392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 987136,
                            "linear_total": 7077888,
                            "nnz": 992628,
                            "total": 7086144
                        },
                        "4": {
                            "linear_attention_nnz": 486400,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 514560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1000960,
                            "linear_total": 7077888,
                            "nnz": 1006479,
                            "total": 7086144
                        },
                        "5": {
                            "linear_attention_nnz": 604160,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 465408,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1069568,
                            "linear_total": 7077888,
                            "nnz": 1075279,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 374784,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 811008,
                            "linear_total": 7077888,
                            "nnz": 816412,
                            "total": 7086144
                        },
                        "7": {
                            "linear_attention_nnz": 398336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 466944,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 865280,
                            "linear_total": 7077888,
                            "nnz": 870768,
                            "total": 7086336
                        },
                        "8": {
                            "linear_attention_nnz": 345088,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 317952,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 663040,
                            "linear_total": 7077888,
                            "nnz": 668367,
                            "total": 7086144
                        },
                        "9": {
                            "linear_attention_nnz": 281600,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 102912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 384512,
                            "linear_total": 7077888,
                            "nnz": 389443,
                            "total": 7086144
                        }
                    },
                    "linear_nnz": 8619520,
                    "linear_sparsity": 89.85158661265432,
                    "linear_total": 84934656,
                    "nnz": 33113492,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9
                        ]
                    },
                    "total": 109464771,
                    "total_sparsity": 69.74963570699838
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.1014841338734569,
            "matched": 80.34640855832909,
            "mismatched": 80.30919446704637,
            "speedup": 14.634450891914463
        },
        {
            "annotate": "9",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8032603158430973,
                    "eval_loss": 0.6140283346176147
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8079739625711961,
                    "eval_loss": 0.580194354057312
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte6_ws12000_rn-output__mnli_test2___fw4_rfl30/checkpoint-55000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 2.9162877101898195,
                    "eval_elapsed_time": 4.395200597122312
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 2.9124035663604735,
                    "eval_elapsed_time": 4.383452212437987
                },
                "speedup": 16.834572109593463,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 334848,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 310272,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 645120,
                            "linear_total": 7077888,
                            "nnz": 650314,
                            "total": 7085952
                        },
                        "1": {
                            "linear_attention_nnz": 508928,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 322560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 831488,
                            "linear_total": 7077888,
                            "nnz": 836946,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 159744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 59904,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 219648,
                            "linear_total": 7077888,
                            "nnz": 224455,
                            "total": 7086144
                        },
                        "11": {
                            "linear_attention_nnz": 111616,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 134656,
                            "linear_total": 7077888,
                            "nnz": 139503,
                            "total": 7086336
                        },
                        "2": {
                            "linear_attention_nnz": 526336,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 402432,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 928768,
                            "linear_total": 7077888,
                            "nnz": 934342,
                            "total": 7086336
                        },
                        "3": {
                            "linear_attention_nnz": 410624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 557568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 968192,
                            "linear_total": 7077888,
                            "nnz": 973675,
                            "total": 7086144
                        },
                        "4": {
                            "linear_attention_nnz": 481280,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 499200,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 980480,
                            "linear_total": 7077888,
                            "nnz": 985989,
                            "total": 7086144
                        },
                        "5": {
                            "linear_attention_nnz": 594944,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 448512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1043456,
                            "linear_total": 7077888,
                            "nnz": 1049156,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 385024,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 416256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 801280,
                            "linear_total": 7077888,
                            "nnz": 806671,
                            "total": 7086144
                        },
                        "7": {
                            "linear_attention_nnz": 396288,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 451584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 847872,
                            "linear_total": 7077888,
                            "nnz": 853350,
                            "total": 7086336
                        },
                        "8": {
                            "linear_attention_nnz": 336896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 634880,
                            "linear_total": 7077888,
                            "nnz": 640194,
                            "total": 7086144
                        },
                        "9": {
                            "linear_attention_nnz": 282624,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 99840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 382464,
                            "linear_total": 7077888,
                            "nnz": 387425,
                            "total": 7086144
                        }
                    },
                    "linear_nnz": 8418304,
                    "linear_sparsity": 90.08849344135803,
                    "linear_total": 84934656,
                    "nnz": 32912103,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9
                        ]
                    },
                    "total": 109464771,
                    "total_sparsity": 69.93361179187046
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.09911506558641969,
            "matched": 80.32603158430973,
            "mismatched": 80.79739625711962,
            "speedup": 16.834572109593463
        },
        {
            "annotate": "9",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.7968415690269995,
                    "eval_loss": 0.6557102203369141
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8062449145646867,
                    "eval_loss": 0.6181047558784485
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte6_ws12000_rn-output__mnli_test2___fw4_rfl30/checkpoint-65000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 2.892913621902466,
                    "eval_elapsed_time": 4.359820336103439
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 2.882220129013062,
                    "eval_elapsed_time": 4.349696953780949
                },
                "speedup": 16.97059164774707,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 331776,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 297984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 629760,
                            "linear_total": 7077888,
                            "nnz": 634946,
                            "total": 7085952
                        },
                        "1": {
                            "linear_attention_nnz": 496640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 311808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 808448,
                            "linear_total": 7077888,
                            "nnz": 813899,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 160768,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 56832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 217600,
                            "linear_total": 7077888,
                            "nnz": 222405,
                            "total": 7086144
                        },
                        "11": {
                            "linear_attention_nnz": 110592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 133632,
                            "linear_total": 7077888,
                            "nnz": 138447,
                            "total": 7086336
                        },
                        "2": {
                            "linear_attention_nnz": 528384,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 379392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 907776,
                            "linear_total": 7077888,
                            "nnz": 913335,
                            "total": 7086336
                        },
                        "3": {
                            "linear_attention_nnz": 394240,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 536064,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 930304,
                            "linear_total": 7077888,
                            "nnz": 935773,
                            "total": 7086144
                        },
                        "4": {
                            "linear_attention_nnz": 464896,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 483840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 948736,
                            "linear_total": 7077888,
                            "nnz": 954235,
                            "total": 7086144
                        },
                        "5": {
                            "linear_attention_nnz": 611328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 422400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1033728,
                            "linear_total": 7077888,
                            "nnz": 1039411,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 368640,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 400896,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 769536,
                            "linear_total": 7077888,
                            "nnz": 774917,
                            "total": 7086144
                        },
                        "7": {
                            "linear_attention_nnz": 408576,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 439296,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 847872,
                            "linear_total": 7077888,
                            "nnz": 853342,
                            "total": 7086336
                        },
                        "8": {
                            "linear_attention_nnz": 337920,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 287232,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 625152,
                            "linear_total": 7077888,
                            "nnz": 630427,
                            "total": 7086144
                        },
                        "9": {
                            "linear_attention_nnz": 278528,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 98304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 376832,
                            "linear_total": 7077888,
                            "nnz": 381760,
                            "total": 7086144
                        }
                    },
                    "linear_nnz": 8229376,
                    "linear_sparsity": 90.31093267746914,
                    "linear_total": 84934656,
                    "nnz": 32722980,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9
                        ]
                    },
                    "total": 109464771,
                    "total_sparsity": 70.10638244517955
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.09689067322530864,
            "matched": 79.68415690269995,
            "mismatched": 80.62449145646868,
            "speedup": 16.97059164774707
        },
        {
            "annotate": "9",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "bert-base-uncased",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8001018848700968,
                    "eval_loss": 0.6599615216255188
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8076688364524003,
                    "eval_loss": 0.6245622634887695
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte6_ws12000_rn-output__mnli_test2___fw4_rfl30/checkpoint-70000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 2.87759080696106,
                    "eval_elapsed_time": 4.352866631932557
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 2.8725717697143556,
                    "eval_elapsed_time": 4.329632396809757
                },
                "speedup": 17.060957948138203,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 349184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 294912,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 644096,
                            "linear_total": 7077888,
                            "nnz": 649280,
                            "total": 7085952
                        },
                        "1": {
                            "linear_attention_nnz": 477184,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 311808,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 788992,
                            "linear_total": 7077888,
                            "nnz": 794443,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 169984,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 56832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 226816,
                            "linear_total": 7077888,
                            "nnz": 231621,
                            "total": 7086144
                        },
                        "11": {
                            "linear_attention_nnz": 110592,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 133632,
                            "linear_total": 7077888,
                            "nnz": 138415,
                            "total": 7086336
                        },
                        "2": {
                            "linear_attention_nnz": 521216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 379392,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 900608,
                            "linear_total": 7077888,
                            "nnz": 906135,
                            "total": 7086336
                        },
                        "3": {
                            "linear_attention_nnz": 382976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 534528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 917504,
                            "linear_total": 7077888,
                            "nnz": 922972,
                            "total": 7086144
                        },
                        "4": {
                            "linear_attention_nnz": 480256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 483840,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 964096,
                            "linear_total": 7077888,
                            "nnz": 969595,
                            "total": 7086144
                        },
                        "5": {
                            "linear_attention_nnz": 595968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 417792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1013760,
                            "linear_total": 7077888,
                            "nnz": 1019440,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 371712,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 396288,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 768000,
                            "linear_total": 7077888,
                            "nnz": 773378,
                            "total": 7086144
                        },
                        "7": {
                            "linear_attention_nnz": 382976,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 819200,
                            "linear_total": 7077888,
                            "nnz": 824668,
                            "total": 7086336
                        },
                        "8": {
                            "linear_attention_nnz": 315392,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 285696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 601088,
                            "linear_total": 7077888,
                            "nnz": 606362,
                            "total": 7086144
                        },
                        "9": {
                            "linear_attention_nnz": 287744,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 98304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 386048,
                            "linear_total": 7077888,
                            "nnz": 390976,
                            "total": 7086144
                        }
                    },
                    "linear_nnz": 8163840,
                    "linear_sparsity": 90.3880931712963,
                    "linear_total": 84934656,
                    "nnz": 32657368,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            5,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            9,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9
                        ]
                    },
                    "total": 109464771,
                    "total_sparsity": 70.16632136379293
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.09611906828703698,
            "matched": 80.01018848700969,
            "mismatched": 80.76688364524003,
            "speedup": 17.060957948138203
        }
    ],
    "Block/struct method, final fine tuned, s=b": [
        {
            "annotate": "25",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8304635761589404,
                    "eval_loss": 0.5648419260978699
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8360455655004069,
                    "eval_loss": 0.5434430241584778
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4/checkpoint-73632",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4/checkpoint-145000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 20.46962840270996,
                    "eval_elapsed_time": 21.966628784313798
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 20.541734985351564,
                    "eval_elapsed_time": 22.030214177444577
                },
                "speedup": 2.398404835869523,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 474624,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1457664,
                            "linear_total": 7077888,
                            "nnz": 1463541,
                            "total": 7086528
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 568320,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1551360,
                            "linear_total": 7077888,
                            "nnz": 1557298,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 109056,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 895488,
                            "linear_total": 7077888,
                            "nnz": 900935,
                            "total": 7086336
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 33792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1606656,
                            "linear_total": 7077888,
                            "nnz": 1612822,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 798720,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2174976,
                            "linear_total": 7077888,
                            "nnz": 2181448,
                            "total": 7086912
                        },
                        "3": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 996864,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2176512,
                            "linear_total": 7077888,
                            "nnz": 2182921,
                            "total": 7086720
                        },
                        "4": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1010688,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2190336,
                            "linear_total": 7077888,
                            "nnz": 2196754,
                            "total": 7086720
                        },
                        "5": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 838656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2411520,
                            "linear_total": 7077888,
                            "nnz": 2418210,
                            "total": 7087104
                        },
                        "6": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 769536,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2145792,
                            "linear_total": 7077888,
                            "nnz": 2152245,
                            "total": 7086912
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 680448,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2056704,
                            "linear_total": 7077888,
                            "nnz": 2063099,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 477696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1657344,
                            "linear_total": 7077888,
                            "nnz": 1663415,
                            "total": 7086720
                        },
                        "9": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 175104,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1158144,
                            "linear_total": 7077888,
                            "nnz": 1163826,
                            "total": 7086528
                        }
                    },
                    "linear_nnz": 21482496,
                    "linear_sparsity": 74.70703125,
                    "linear_total": 84934656,
                    "nnz": 45986597,
                    "pruned_heads": {
                        "0": [
                            0,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11,
                            6
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109471107,
                    "total_sparsity": 57.99202341125499
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.2529296875,
            "matched": 83.04635761589404,
            "mismatched": 83.60455655004068,
            "speedup": 2.398404835869523
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8268976057055527,
                    "eval_loss": 0.5891618132591248
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8271969080553295,
                    "eval_loss": 0.5817938446998596
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20/checkpoint-70000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 17.147825485229493,
                    "eval_elapsed_time": 18.60521282814443
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 16.814879684448243,
                    "eval_elapsed_time": 18.297247163951397
                },
                "speedup": 2.8630134935651097,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 268800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1055232,
                            "linear_total": 7077888,
                            "nnz": 1060783,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1090560,
                            "linear_total": 7077888,
                            "nnz": 1096134,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 70656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 857088,
                            "linear_total": 7077888,
                            "nnz": 862510,
                            "total": 7086336
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1006080,
                            "linear_total": 7077888,
                            "nnz": 1011663,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 459264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1245696,
                            "linear_total": 7077888,
                            "nnz": 1251371,
                            "total": 7086336
                        },
                        "3": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1597440,
                            "linear_total": 7077888,
                            "nnz": 1603408,
                            "total": 7086528
                        },
                        "4": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 602112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1585152,
                            "linear_total": 7077888,
                            "nnz": 1591112,
                            "total": 7086528
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 489984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1866240,
                            "linear_total": 7077888,
                            "nnz": 1872511,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1419264,
                            "linear_total": 7077888,
                            "nnz": 1425116,
                            "total": 7086528
                        },
                        "7": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 453120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1436160,
                            "linear_total": 7077888,
                            "nnz": 1442023,
                            "total": 7086528
                        },
                        "8": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 327168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1310208,
                            "linear_total": 7077888,
                            "nnz": 1315989,
                            "total": 7086528
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 112128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 898560,
                            "linear_total": 7077888,
                            "nnz": 904009,
                            "total": 7086336
                        }
                    },
                    "linear_nnz": 15367680,
                    "linear_sparsity": 81.90646701388889,
                    "linear_total": 84934656,
                    "nnz": 39866712,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109467843,
                    "total_sparsity": 63.581348725396914
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.18093532986111116,
            "matched": 82.68976057055526,
            "mismatched": 82.71969080553295,
            "speedup": 2.8630134935651097
        },
        {
            "annotate": "18",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8267957208354559,
                    "eval_loss": 0.5853732824325562
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8275020341741253,
                    "eval_loss": 0.5784305930137634
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20/checkpoint-73632",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20/checkpoint-100000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 16.80277032470703,
                    "eval_elapsed_time": 18.324330019764602
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 16.830738342285155,
                    "eval_elapsed_time": 18.355786813423038
                },
                "speedup": 2.9218072258788506,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 268800,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1055232,
                            "linear_total": 7077888,
                            "nnz": 1060783,
                            "total": 7086336
                        },
                        "1": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 304128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1090560,
                            "linear_total": 7077888,
                            "nnz": 1096134,
                            "total": 7086336
                        },
                        "10": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 70656,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 857088,
                            "linear_total": 7077888,
                            "nnz": 862510,
                            "total": 7086336
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1006080,
                            "linear_total": 7077888,
                            "nnz": 1011663,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 459264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1245696,
                            "linear_total": 7077888,
                            "nnz": 1251371,
                            "total": 7086336
                        },
                        "3": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 614400,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1597440,
                            "linear_total": 7077888,
                            "nnz": 1603408,
                            "total": 7086528
                        },
                        "4": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 602112,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1585152,
                            "linear_total": 7077888,
                            "nnz": 1591112,
                            "total": 7086528
                        },
                        "5": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 489984,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1866240,
                            "linear_total": 7077888,
                            "nnz": 1872511,
                            "total": 7086912
                        },
                        "6": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 436224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1419264,
                            "linear_total": 7077888,
                            "nnz": 1425116,
                            "total": 7086528
                        },
                        "7": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 453120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1436160,
                            "linear_total": 7077888,
                            "nnz": 1442023,
                            "total": 7086528
                        },
                        "8": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 327168,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1310208,
                            "linear_total": 7077888,
                            "nnz": 1315989,
                            "total": 7086528
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 112128,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 898560,
                            "linear_total": 7077888,
                            "nnz": 904009,
                            "total": 7086336
                        }
                    },
                    "linear_nnz": 15367680,
                    "linear_sparsity": 81.90646701388889,
                    "linear_total": 84934656,
                    "nnz": 39866712,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            0,
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            5,
                            6,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109467843,
                    "total_sparsity": 63.581348725396914
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl20",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.18093532986111116,
            "matched": 82.6795720835456,
            "mismatched": 82.75020341741252,
            "speedup": 2.9218072258788506
        },
        {
            "annotate": "12",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8102903718797758,
                    "eval_loss": 0.6519557237625122
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8132628152969894,
                    "eval_loss": 0.63445645570755
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl40/checkpoint-73632",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl40/checkpoint-140000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 14.28297381591797,
                    "eval_elapsed_time": 15.819611034356058
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 13.850817779541016,
                    "eval_elapsed_time": 15.406974045559764
                },
                "speedup": 3.4372712841353343,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 393216,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 93696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 486912,
                            "linear_total": 7077888,
                            "nnz": 491965,
                            "total": 7085952
                        },
                        "1": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 98304,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 688128,
                            "linear_total": 7077888,
                            "nnz": 693376,
                            "total": 7086144
                        },
                        "10": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 46080,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 635904,
                            "linear_total": 7077888,
                            "nnz": 641118,
                            "total": 7086144
                        },
                        "11": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 23040,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1006080,
                            "linear_total": 7077888,
                            "nnz": 1011663,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 192000,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 978432,
                            "linear_total": 7077888,
                            "nnz": 983933,
                            "total": 7086336
                        },
                        "3": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 285696,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1072128,
                            "linear_total": 7077888,
                            "nnz": 1077690,
                            "total": 7086336
                        },
                        "4": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 869376,
                            "linear_total": 7077888,
                            "nnz": 874742,
                            "total": 7086144
                        },
                        "5": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 225792,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1208832,
                            "linear_total": 7077888,
                            "nnz": 1214547,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 213504,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 803328,
                            "linear_total": 7077888,
                            "nnz": 808651,
                            "total": 7086144
                        },
                        "7": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 254976,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1041408,
                            "linear_total": 7077888,
                            "nnz": 1046950,
                            "total": 7086336
                        },
                        "8": {
                            "linear_attention_nnz": 589824,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 199680,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 789504,
                            "linear_total": 7077888,
                            "nnz": 794818,
                            "total": 7086144
                        },
                        "9": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 69120,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 855552,
                            "linear_total": 7077888,
                            "nnz": 860973,
                            "total": 7086336
                        }
                    },
                    "linear_nnz": 10435584,
                    "linear_sparsity": 87.71339699074075,
                    "linear_total": 84934656,
                    "nnz": 34930509,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            2,
                            6,
                            7,
                            10,
                            11
                        ],
                        "2": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            1,
                            2,
                            3,
                            4,
                            6,
                            7,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            10,
                            11
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109465155,
                    "total_sparsity": 68.08983735509258
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl40",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl40",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl40",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.12286603009259256,
            "matched": 81.02903718797758,
            "mismatched": 81.32628152969895,
            "speedup": 3.4372712841353343
        },
        {
            "annotate": "34",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8370860927152318,
                    "eval_loss": 0.5330445170402527
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8408258746948739,
                    "eval_loss": 0.5096385478973389
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-65000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.553811492919923,
                    "eval_elapsed_time": 26.061392509378493
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 24.65311068725586,
                    "eval_elapsed_time": 26.166672149673104
                },
                "speedup": 1.9994637396181068,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 970752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2347008,
                            "linear_total": 7077888,
                            "nnz": 2353592,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1047552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2030592,
                            "linear_total": 7077888,
                            "nnz": 2036842,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 952320,
                            "linear_total": 7077888,
                            "nnz": 957804,
                            "total": 7086336
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 56832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1629696,
                            "linear_total": 7077888,
                            "nnz": 1635877,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1325568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2701824,
                            "linear_total": 7077888,
                            "nnz": 2708639,
                            "total": 7086912
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1611264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3184128,
                            "linear_total": 7077888,
                            "nnz": 3191321,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1666560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3436032,
                            "linear_total": 7077888,
                            "nnz": 3443453,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1433088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3202560,
                            "linear_total": 7077888,
                            "nnz": 3209829,
                            "total": 7087296
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3050496,
                            "linear_total": 7077888,
                            "nnz": 3057666,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1118208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2494464,
                            "linear_total": 7077888,
                            "nnz": 2501144,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 800256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2569728,
                            "linear_total": 7077888,
                            "nnz": 2576585,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1459200,
                            "linear_total": 7077888,
                            "nnz": 1465142,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 29058048,
                    "linear_sparsity": 65.78776041666667,
                    "linear_total": 84934656,
                    "nnz": 53567977,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109473795,
                    "total_sparsity": 51.0677628376727
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.34212239583333326,
            "matched": 83.70860927152319,
            "mismatched": 84.0825874694874,
            "speedup": 1.9994637396181068
        },
        {
            "annotate": "34",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8361691288843607,
                    "eval_loss": 0.5352884531021118
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8403173311635476,
                    "eval_loss": 0.5110790729522705
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-70000",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.5699239654541,
                    "eval_elapsed_time": 26.07937575969845
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 24.643021484375,
                    "eval_elapsed_time": 26.15799339208752
                },
                "speedup": 1.9981525306525039,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 970752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2347008,
                            "linear_total": 7077888,
                            "nnz": 2353592,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1047552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2030592,
                            "linear_total": 7077888,
                            "nnz": 2036842,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 952320,
                            "linear_total": 7077888,
                            "nnz": 957804,
                            "total": 7086336
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 56832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1629696,
                            "linear_total": 7077888,
                            "nnz": 1635877,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1325568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2701824,
                            "linear_total": 7077888,
                            "nnz": 2708639,
                            "total": 7086912
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1611264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3184128,
                            "linear_total": 7077888,
                            "nnz": 3191321,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1666560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3436032,
                            "linear_total": 7077888,
                            "nnz": 3443453,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1433088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3202560,
                            "linear_total": 7077888,
                            "nnz": 3209829,
                            "total": 7087296
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3050496,
                            "linear_total": 7077888,
                            "nnz": 3057666,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1118208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2494464,
                            "linear_total": 7077888,
                            "nnz": 2501144,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 800256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2569728,
                            "linear_total": 7077888,
                            "nnz": 2576585,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1459200,
                            "linear_total": 7077888,
                            "nnz": 1465142,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 29058048,
                    "linear_sparsity": 65.78776041666667,
                    "linear_total": 84934656,
                    "nnz": 53567977,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109473795,
                    "total_sparsity": 51.0677628376727
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.34212239583333326,
            "matched": 83.61691288843606,
            "mismatched": 84.03173311635476,
            "speedup": 1.9981525306525039
        },
        {
            "annotate": "34",
            "cat_fun_name": "is_new_xp",
            "checkpoint": {
                "config": {
                    "_name_or_path": "/home/lagunas/devel/hf/nn_pruning/nn_pruning/analysis/tmp_finetune",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8360672440142639,
                    "eval_loss": 0.5312635898590088
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8409275834011392,
                    "eval_loss": 0.5076500177383423
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-73632",
                "source_checkpoint": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5/checkpoint-135000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 1,
                    "attention_block_rows": 1,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "topK",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "topK",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": 1,
                    "final_threshold": 0.5,
                    "final_warmup": 0,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 1.0,
                    "initial_warmup": 0,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "",
                    "regularization_final_lambda": 0
                },
                "speed": {
                    "cuda_eval_elapsed_time": 24.566403701782228,
                    "eval_elapsed_time": 26.104158860631287
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 24.638316284179687,
                    "eval_elapsed_time": 26.13123631104827
                },
                "speedup": 1.9984388576155345,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 970752,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2347008,
                            "linear_total": 7077888,
                            "nnz": 2353592,
                            "total": 7086912
                        },
                        "1": {
                            "linear_attention_nnz": 983040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1047552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2030592,
                            "linear_total": 7077888,
                            "nnz": 2036842,
                            "total": 7086528
                        },
                        "10": {
                            "linear_attention_nnz": 786432,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 165888,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 952320,
                            "linear_total": 7077888,
                            "nnz": 957804,
                            "total": 7086336
                        },
                        "11": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 56832,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1629696,
                            "linear_total": 7077888,
                            "nnz": 1635877,
                            "total": 7087104
                        },
                        "2": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1325568,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2701824,
                            "linear_total": 7077888,
                            "nnz": 2708639,
                            "total": 7086912
                        },
                        "3": {
                            "linear_attention_nnz": 1572864,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1611264,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3184128,
                            "linear_total": 7077888,
                            "nnz": 3191321,
                            "total": 7087104
                        },
                        "4": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1666560,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3436032,
                            "linear_total": 7077888,
                            "nnz": 3443453,
                            "total": 7087296
                        },
                        "5": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1433088,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3202560,
                            "linear_total": 7077888,
                            "nnz": 3209829,
                            "total": 7087296
                        },
                        "6": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1281024,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 3050496,
                            "linear_total": 7077888,
                            "nnz": 3057666,
                            "total": 7087296
                        },
                        "7": {
                            "linear_attention_nnz": 1376256,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 1118208,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2494464,
                            "linear_total": 7077888,
                            "nnz": 2501144,
                            "total": 7086912
                        },
                        "8": {
                            "linear_attention_nnz": 1769472,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 800256,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 2569728,
                            "linear_total": 7077888,
                            "nnz": 2576585,
                            "total": 7087296
                        },
                        "9": {
                            "linear_attention_nnz": 1179648,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 279552,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 1459200,
                            "linear_total": 7077888,
                            "nnz": 1465142,
                            "total": 7086720
                        }
                    },
                    "linear_nnz": 29058048,
                    "linear_sparsity": 65.78776041666667,
                    "linear_total": 84934656,
                    "nnz": 53567977,
                    "pruned_heads": {
                        "0": [
                            2,
                            4,
                            7,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8
                        ],
                        "10": [
                            0,
                            1,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            1,
                            11,
                            7
                        ],
                        "2": [
                            4,
                            5,
                            7,
                            8,
                            11
                        ],
                        "3": [
                            2,
                            4,
                            6,
                            7
                        ],
                        "4": [
                            8,
                            1,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            11
                        ],
                        "6": [
                            11,
                            10,
                            3
                        ],
                        "7": [
                            2,
                            4,
                            6,
                            7,
                            11
                        ],
                        "8": [
                            0,
                            10,
                            6
                        ],
                        "9": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109473795,
                    "total_sparsity": 51.0677628376727
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 6,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "/data_2to/devel_data/nn_pruning/output/mnli_test_final_fine_tune/fine_tuned_hp_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl5",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 10,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.34212239583333326,
            "matched": 83.60672440142639,
            "mismatched": 84.09275834011392,
            "speedup": 1.9984388576155345
        }
    ],
    "short final warmup": [
        {
            "annotate": "",
            "cat_fun_name": "is_short_final_warmup",
            "checkpoint": {
                "config": {
                    "_name_or_path": "aloxatel/bert-base-mnli",
                    "architectures": [
                        "BertForSequenceClassification"
                    ],
                    "attention_probs_dropout_prob": 0.1,
                    "finetuning_task": "mnli",
                    "gradient_checkpointing": false,
                    "hidden_act": "gelu",
                    "hidden_dropout_prob": 0.1,
                    "hidden_size": 768,
                    "id2label": {
                        "0": "contradiction",
                        "1": "entailment",
                        "2": "neutral"
                    },
                    "initializer_range": 0.02,
                    "intermediate_size": 3072,
                    "label2id": {
                        "contradiction": 0,
                        "entailment": 1,
                        "neutral": 2
                    },
                    "layer_norm_eps": 1e-12,
                    "max_position_embeddings": 512,
                    "model_type": "bert",
                    "num_attention_heads": 12,
                    "num_hidden_layers": 12,
                    "pad_token_id": 0,
                    "position_embedding_type": "absolute",
                    "type_vocab_size": 2,
                    "vocab_size": 30522
                },
                "eval_metrics": {
                    "eval_accuracy": 0.8071319409067753,
                    "eval_loss": 0.6617238521575928
                },
                "eval_metrics_mm": {
                    "eval_accuracy": 0.8083807973962571,
                    "eval_loss": 0.6353434920310974
                },
                "path": "/data_2to/devel_data/nn_pruning/output/mnli_test2/hp_mnop-aloxatel__bert-base-mnli_od-output__mnli_test2___pdtbs32_pdebs128_nte12_ws12000_rn-output__mnli_test2___fw4_rfl30/checkpoint-145000",
                "sparse_args": {
                    "ampere_pruning_method": "disabled",
                    "attention_block_cols": 32,
                    "attention_block_rows": 32,
                    "attention_lambda": 1.0,
                    "attention_output_with_dense": 0,
                    "attention_pruning_method": "sigmoied_threshold",
                    "bias_mask": true,
                    "dense_block_cols": 1,
                    "dense_block_rows": 1,
                    "dense_lambda": 1.0,
                    "dense_pruning_method": "sigmoied_threshold:1d_alt",
                    "distil_alpha_ce": 0.1,
                    "distil_alpha_teacher": 0.9,
                    "distil_teacher_name_or_path": "aloxatel/bert-base-mnli",
                    "distil_temperature": 2.0,
                    "final_ampere_temperature": 20.0,
                    "final_finetune": false,
                    "final_threshold": 0.1,
                    "final_warmup": 4,
                    "initial_ampere_temperature": 0.0,
                    "initial_threshold": 0,
                    "initial_warmup": 1,
                    "mask_init": "constant",
                    "mask_scale": 0.0,
                    "mask_scores_learning_rate": 0.01,
                    "regularization": "l1",
                    "regularization_final_lambda": 30
                },
                "speed": {
                    "cuda_eval_elapsed_time": 2.85058770942688,
                    "eval_elapsed_time": 4.325579055584967
                },
                "speed_mm": {
                    "cuda_eval_elapsed_time": 2.844103899002075,
                    "eval_elapsed_time": 4.3065507067367435
                },
                "speedup": 17.22257329152041,
                "stats": {
                    "layers": {
                        "0": {
                            "linear_attention_nnz": 424960,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 101376,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 526336,
                            "linear_total": 7077888,
                            "nnz": 531490,
                            "total": 7086144
                        },
                        "1": {
                            "linear_attention_nnz": 362496,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 148992,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 511488,
                            "linear_total": 7077888,
                            "nnz": 516641,
                            "total": 7086144
                        },
                        "10": {
                            "linear_attention_nnz": 263168,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 52224,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 315392,
                            "linear_total": 7077888,
                            "nnz": 320130,
                            "total": 7086144
                        },
                        "11": {
                            "linear_attention_nnz": 131072,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 29184,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 160256,
                            "linear_total": 7077888,
                            "nnz": 165139,
                            "total": 7086528
                        },
                        "2": {
                            "linear_attention_nnz": 525312,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 259584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 784896,
                            "linear_total": 7077888,
                            "nnz": 790281,
                            "total": 7086336
                        },
                        "3": {
                            "linear_attention_nnz": 375808,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 342528,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 718336,
                            "linear_total": 7077888,
                            "nnz": 723647,
                            "total": 7086144
                        },
                        "4": {
                            "linear_attention_nnz": 518144,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 334848,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 852992,
                            "linear_total": 7077888,
                            "nnz": 858490,
                            "total": 7086336
                        },
                        "5": {
                            "linear_attention_nnz": 471040,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 259584,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 730624,
                            "linear_total": 7077888,
                            "nnz": 736105,
                            "total": 7086528
                        },
                        "6": {
                            "linear_attention_nnz": 264192,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 256512,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 520704,
                            "linear_total": 7077888,
                            "nnz": 525927,
                            "total": 7086144
                        },
                        "7": {
                            "linear_attention_nnz": 339968,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 278016,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 617984,
                            "linear_total": 7077888,
                            "nnz": 623253,
                            "total": 7086336
                        },
                        "8": {
                            "linear_attention_nnz": 248832,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 196608,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 445440,
                            "linear_total": 7077888,
                            "nnz": 450656,
                            "total": 7086144
                        },
                        "9": {
                            "linear_attention_nnz": 355328,
                            "linear_attention_total": 2359296,
                            "linear_dense_nnz": 96768,
                            "linear_dense_total": 4718592,
                            "linear_nnz": 452096,
                            "linear_total": 7077888,
                            "nnz": 457279,
                            "total": 7086336
                        }
                    },
                    "linear_nnz": 6636544,
                    "linear_sparsity": 92.18629436728395,
                    "linear_total": 84934656,
                    "nnz": 31129121,
                    "pruned_heads": {
                        "0": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            7,
                            8,
                            9,
                            11
                        ],
                        "1": [
                            0,
                            1,
                            2,
                            3,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "10": [
                            0,
                            1,
                            2,
                            4,
                            5,
                            6,
                            7,
                            8,
                            9
                        ],
                        "11": [
                            0,
                            3,
                            5,
                            6,
                            7,
                            8,
                            11
                        ],
                        "2": [
                            1,
                            3,
                            4,
                            5,
                            7,
                            8,
                            9,
                            11
                        ],
                        "3": [
                            1,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "4": [
                            0,
                            1,
                            2,
                            4,
                            6,
                            8,
                            10,
                            11
                        ],
                        "5": [
                            1,
                            2,
                            4,
                            5,
                            6,
                            9,
                            11
                        ],
                        "6": [
                            2,
                            3,
                            4,
                            6,
                            7,
                            8,
                            9,
                            10,
                            11
                        ],
                        "7": [
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            9,
                            11
                        ],
                        "8": [
                            0,
                            2,
                            3,
                            4,
                            5,
                            6,
                            7,
                            8,
                            10
                        ],
                        "9": [
                            0,
                            1,
                            2,
                            3,
                            4,
                            5,
                            7,
                            9
                        ]
                    },
                    "total": 109465347,
                    "total_sparsity": 71.56257952573795
                },
                "training_args": {
                    "_n_gpu": -1,
                    "adafactor": false,
                    "adam_beta1": 0.9,
                    "adam_beta2": 0.999,
                    "adam_epsilon": 1e-08,
                    "dataloader_drop_last": false,
                    "dataloader_num_workers": 0,
                    "dataloader_pin_memory": true,
                    "ddp_find_unused_parameters": null,
                    "debug": false,
                    "deepspeed": null,
                    "disable_tqdm": false,
                    "do_eval": 1,
                    "do_predict": false,
                    "do_train": 1,
                    "eval_accumulation_steps": null,
                    "eval_steps": 5000,
                    "evaluation_strategy": "steps",
                    "fp16": false,
                    "fp16_backend": "auto",
                    "fp16_full_eval": false,
                    "fp16_opt_level": "O1",
                    "gradient_accumulation_steps": 1,
                    "greater_is_better": null,
                    "group_by_length": false,
                    "ignore_data_skip": false,
                    "label_names": null,
                    "label_smoothing_factor": 0.0,
                    "learning_rate": 3e-05,
                    "load_best_model_at_end": false,
                    "local_rank": -1,
                    "logging_dir": "output/mnli_test2/",
                    "logging_first_step": false,
                    "logging_steps": 250,
                    "logging_strategy": "steps",
                    "lr_scheduler_type": "linear",
                    "max_grad_norm": 1.0,
                    "max_steps": -1,
                    "metric_for_best_model": null,
                    "no_cuda": false,
                    "num_train_epochs": 12,
                    "optimize_model_before_eval": "disabled",
                    "output_dir": "output/mnli_test2/",
                    "overwrite_output_dir": 1,
                    "past_index": -1,
                    "per_device_eval_batch_size": 128,
                    "per_device_train_batch_size": 32,
                    "per_gpu_eval_batch_size": null,
                    "per_gpu_train_batch_size": null,
                    "prediction_loss_only": false,
                    "remove_unused_columns": true,
                    "report_to": null,
                    "run_name": "output/mnli_test2/",
                    "save_steps": 5000,
                    "save_strategy": "steps",
                    "save_total_limit": 50,
                    "seed": 17,
                    "sharded_ddp": "",
                    "skip_memory_metrics": false,
                    "tpu_metrics_debug": false,
                    "tpu_num_cores": null,
                    "warmup_ratio": 0.0,
                    "warmup_steps": 12000,
                    "weight_decay": 0.0
                }
            },
            "fill_rate": 0.0781370563271605,
            "matched": 80.71319409067753,
            "mismatched": 80.83807973962571,
            "speedup": 17.22257329152041
        }
    ]
}