{"fill_rate": 0.38938319830246915, "f1": 89.74014850499854, "meta": {"fill_rate": 0.38938319830246915, "speedup": 1.550702144203815, "checkpoint": {"config": {"_name_or_path": "bert-base-uncased", "architectures": ["BertForQuestionAnswering"], "attention_probs_dropout_prob": 0.1, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-12, "max_position_embeddings": 512, "model_type": "bert", "num_attention_heads": 12, "num_hidden_layers": 12, "pad_token_id": 0, "position_embedding_type": "absolute", "type_vocab_size": 2, "vocab_size": 30522}, "eval_metrics": {"exact_match": 82.94228949858089, "f1": 89.74014850499854}, "sparse_args": {"ampere_pruning_method": "disabled", "attention_block_cols": 32, "attention_block_rows": 32, "attention_lambda": 1.0, "attention_output_with_dense": 0, "attention_pruning_method": "sigmoied_threshold", "bias_mask": true, "dense_block_cols": 1, "dense_block_rows": 1, "dense_lambda": 1.0, "dense_pruning_method": "sigmoied_threshold:1d_alt", "distil_alpha_ce": 0.1, "distil_alpha_teacher": 0.9, "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad", "distil_temperature": 2.0, "final_ampere_temperature": 20.0, "final_finetune": false, "final_threshold": 0.1, "final_warmup": 10, "initial_ampere_temperature": 0.0, "initial_threshold": 0, "initial_warmup": 1, "mask_init": "constant", "mask_scale": 0.0, "mask_scores_learning_rate": 0.01, "regularization": "l1", "regularization_final_lambda": 2.5}, "speed": {"cuda_eval_elapsed_time": 24.88833406829834, "eval_elapsed_time": 31.90413049608469}, "speedup": 1.550702144203815, "stats": {"layers": {"0": {"linear_attention_nnz": 1664000, "linear_attention_total": 2359296, "linear_dense_nnz": 1720320, "linear_dense_total": 4718592, "linear_nnz": 3384320, "linear_total": 7077888, "nnz": 3391936, "total": 7087872}, "1": {"linear_attention_nnz": 1323008, "linear_attention_total": 2359296, "linear_dense_nnz": 1815552, "linear_dense_total": 4718592, "linear_nnz": 3138560, "linear_total": 7077888, "nnz": 3146014, "total": 7087872}, "2": {"linear_attention_nnz": 1634304, "linear_attention_total": 2359296, "linear_dense_nnz": 1961472, "linear_dense_total": 4718592, "linear_nnz": 3595776, "linear_total": 7077888, "nnz": 3603517, "total": 7087872}, "3": {"linear_attention_nnz": 1774592, "linear_attention_total": 2359296, "linear_dense_nnz": 1981440, "linear_dense_total": 4718592, "linear_nnz": 3756032, "linear_total": 7077888, "nnz": 3763850, "total": 7087872}, "4": {"linear_attention_nnz": 2001920, "linear_attention_total": 2359296, "linear_dense_nnz": 1850880, "linear_dense_total": 4718592, "linear_nnz": 3852800, "linear_total": 7077888, "nnz": 3860821, "total": 7087872}, "5": {"linear_attention_nnz": 1760256, "linear_attention_total": 2359296, "linear_dense_nnz": 1752576, "linear_dense_total": 4718592, "linear_nnz": 3512832, "linear_total": 7077888, "nnz": 3520597, "total": 7087872}, "6": {"linear_attention_nnz": 1860608, "linear_attention_total": 2359296, "linear_dense_nnz": 1350144, "linear_dense_total": 4718592, "linear_nnz": 3210752, "linear_total": 7077888, "nnz": 3218351, "total": 7087872}, "7": {"linear_attention_nnz": 1329152, "linear_attention_total": 2359296, "linear_dense_nnz": 1013760, "linear_dense_total": 4718592, "linear_nnz": 2342912, "linear_total": 7077888, "nnz": 2349684, "total": 7087872}, "8": {"linear_attention_nnz": 1685504, "linear_attention_total": 2359296, "linear_dense_nnz": 511488, "linear_dense_total": 4718592, "linear_nnz": 2196992, "linear_total": 7077888, "nnz": 2204045, "total": 7087872}, "9": {"linear_attention_nnz": 1032192, "linear_attention_total": 2359296, "linear_dense_nnz": 254976, "linear_dense_total": 4718592, "linear_nnz": 1287168, "linear_total": 7077888, "nnz": 1293286, "total": 7087872}, "10": {"linear_attention_nnz": 966656, "linear_attention_total": 2359296, "linear_dense_nnz": 391680, "linear_dense_total": 4718592, "linear_nnz": 1358336, "linear_total": 7077888, "nnz": 1364639, "total": 7087872}, "11": {"linear_attention_nnz": 796672, "linear_attention_total": 2359296, "linear_dense_nnz": 638976, "linear_dense_total": 4718592, "linear_nnz": 1435648, "linear_total": 7077888, "nnz": 1442016, "total": 7087872}}, "linear_nnz": 33072128, "linear_sparsity": 61.06168016975309, "linear_total": 84934656, "nnz": 56997478, "pruned_heads": {"0": [0], "1": [0, 2, 3], "2": [8, 4], "3": [2, 4], "4": [], "5": [1, 2], "6": [3], "7": [11, 3, 6, 7], "8": [], "9": [1, 4, 5, 7], "10": [1, 4, 7], "11": [8, 5, 7]}, "total": 108893186, "total_sparsity": 47.65744295515424}, "training_args": {"adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "dataloader_drop_last": false, "dataloader_num_workers": 0, "debug": false, "disable_tqdm": false, "do_eval": 1, "do_predict": false, "do_train": 1, "eval_accumulation_steps": null, "eval_steps": 5000, "evaluation_strategy": "steps", "fp16": false, "fp16_opt_level": "O1", "gradient_accumulation_steps": 1, "greater_is_better": null, "ignore_data_skip": false, "label_names": null, "learning_rate": 3e-05, "load_best_model_at_end": false, "local_rank": -1, "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "logging_first_step": false, "logging_steps": 250, "max_grad_norm": 1.0, "max_steps": -1, "metric_for_best_model": null, "model_parallel": false, "no_cuda": false, "num_train_epochs": 20, "optimize_model_before_eval": "disabled", "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "overwrite_output_dir": 1, "past_index": -1, "per_device_eval_batch_size": 8, "per_device_train_batch_size": 16, "per_gpu_eval_batch_size": null, "per_gpu_train_batch_size": null, "prediction_loss_only": false, "remove_unused_columns": true, "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "save_steps": 5000, "save_total_limit": 50, "seed": 17, "tpu_metrics_debug": false, "tpu_num_cores": null, "warmup_steps": 5400, "weight_decay": 0.0}, "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold:--d169c0ebde721c7/checkpoint-110000"}, "f1": 89.74014850499854, "cat_fun_name": "is_new_xp", "annotate": "38"}}
{"fill_rate": 0.29069613233024694, "f1": 88.96065210705885, "meta": {"fill_rate": 0.29069613233024694, "speedup": 1.816892934733715, "checkpoint": {"config": {"_name_or_path": "bert-base-uncased", "architectures": ["BertForQuestionAnswering"], "attention_probs_dropout_prob": 0.1, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-12, "max_position_embeddings": 512, "model_type": "bert", "num_attention_heads": 12, "num_hidden_layers": 12, "pad_token_id": 0, "position_embedding_type": "absolute", "type_vocab_size": 2, "vocab_size": 30522}, "eval_metrics": {"exact_match": 81.86376537369915, "f1": 88.96065210705885}, "sparse_args": {"ampere_pruning_method": "disabled", "attention_block_cols": 32, "attention_block_rows": 32, "attention_lambda": 1.0, "attention_output_with_dense": 0, "attention_pruning_method": "sigmoied_threshold", "bias_mask": true, "dense_block_cols": 1, "dense_block_rows": 1, "dense_lambda": 1.0, "dense_pruning_method": "sigmoied_threshold:1d_alt", "distil_alpha_ce": 0.1, "distil_alpha_teacher": 0.9, "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad", "distil_temperature": 2.0, "final_ampere_temperature": 20.0, "final_finetune": false, "final_threshold": 0.1, "final_warmup": 10, "initial_ampere_temperature": 0.0, "initial_threshold": 0, "initial_warmup": 1, "mask_init": "constant", "mask_scale": 0.0, "mask_scores_learning_rate": 0.01, "regularization": "l1", "regularization_final_lambda": 5}, "speed": {"cuda_eval_elapsed_time": 21.24197428894043, "eval_elapsed_time": 28.05219709314406}, "speedup": 1.816892934733715, "stats": {"layers": {"0": {"linear_attention_nnz": 1262592, "linear_attention_total": 2359296, "linear_dense_nnz": 964608, "linear_dense_total": 4718592, "linear_nnz": 2227200, "linear_total": 7077888, "nnz": 2233876, "total": 7087872}, "1": {"linear_attention_nnz": 850944, "linear_attention_total": 2359296, "linear_dense_nnz": 1202688, "linear_dense_total": 4718592, "linear_nnz": 2053632, "linear_total": 7077888, "nnz": 2060047, "total": 7087872}, "10": {"linear_attention_nnz": 732160, "linear_attention_total": 2359296, "linear_dense_nnz": 256512, "linear_dense_total": 4718592, "linear_nnz": 988672, "linear_total": 7077888, "nnz": 994567, "total": 7087872}, "11": {"linear_attention_nnz": 600064, "linear_attention_total": 2359296, "linear_dense_nnz": 400896, "linear_dense_total": 4718592, "linear_nnz": 1000960, "linear_total": 7077888, "nnz": 1006917, "total": 7087872}, "2": {"linear_attention_nnz": 1501184, "linear_attention_total": 2359296, "linear_dense_nnz": 1281024, "linear_dense_total": 4718592, "linear_nnz": 2782208, "linear_total": 7077888, "nnz": 2789378, "total": 7087872}, "3": {"linear_attention_nnz": 1546240, "linear_attention_total": 2359296, "linear_dense_nnz": 1319424, "linear_dense_total": 4718592, "linear_nnz": 2865664, "linear_total": 7077888, "nnz": 2872859, "total": 7087872}, "4": {"linear_attention_nnz": 1752064, "linear_attention_total": 2359296, "linear_dense_nnz": 1288704, "linear_dense_total": 4718592, "linear_nnz": 3040768, "linear_total": 7077888, "nnz": 3048199, "total": 7087872}, "5": {"linear_attention_nnz": 1638400, "linear_attention_total": 2359296, "linear_dense_nnz": 1210368, "linear_dense_total": 4718592, "linear_nnz": 2848768, "linear_total": 7077888, "nnz": 2856084, "total": 7087872}, "6": {"linear_attention_nnz": 1496064, "linear_attention_total": 2359296, "linear_dense_nnz": 940032, "linear_dense_total": 4718592, "linear_nnz": 2436096, "linear_total": 7077888, "nnz": 2443172, "total": 7087872}, "7": {"linear_attention_nnz": 1236992, "linear_attention_total": 2359296, "linear_dense_nnz": 662016, "linear_dense_total": 4718592, "linear_nnz": 1899008, "linear_total": 7077888, "nnz": 1905519, "total": 7087872}, "8": {"linear_attention_nnz": 1231872, "linear_attention_total": 2359296, "linear_dense_nnz": 348672, "linear_dense_total": 4718592, "linear_nnz": 1580544, "linear_total": 7077888, "nnz": 1587011, "total": 7087872}, "9": {"linear_attention_nnz": 806912, "linear_attention_total": 2359296, "linear_dense_nnz": 159744, "linear_dense_total": 4718592, "linear_nnz": 966656, "linear_total": 7077888, "nnz": 972488, "total": 7087872}}, "linear_nnz": 24690176, "linear_sparsity": 70.9303867669753, "linear_total": 84934656, "nnz": 48608839, "pruned_heads": {"0": [0, 4, 5, 6], "1": [0, 2, 3, 5, 6, 8], "10": [1, 4, 5, 6, 7], "11": [8, 10, 5, 7], "2": [8, 4], "3": [2, 4, 6], "4": [1], "5": [1, 2], "6": [2, 3], "7": [11, 3, 6, 7], "8": [0, 3, 4], "9": [1, 4, 5, 7, 10]}, "total": 108893186, "total_sparsity": 55.3609910908475}, "training_args": {"adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "dataloader_drop_last": false, "dataloader_num_workers": 0, "debug": false, "disable_tqdm": false, "do_eval": 1, "do_predict": false, "do_train": 1, "eval_accumulation_steps": null, "eval_steps": 5000, "evaluation_strategy": "steps", "fp16": false, "fp16_opt_level": "O1", "gradient_accumulation_steps": 1, "greater_is_better": null, "ignore_data_skip": false, "label_names": null, "learning_rate": 3e-05, "load_best_model_at_end": false, "local_rank": -1, "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "logging_first_step": false, "logging_steps": 250, "max_grad_norm": 1.0, "max_steps": -1, "metric_for_best_model": null, "model_parallel": false, "no_cuda": false, "num_train_epochs": 20, "optimize_model_before_eval": "disabled", "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "overwrite_output_dir": 1, "past_index": -1, "per_device_eval_batch_size": 8, "per_device_train_batch_size": 16, "per_gpu_eval_batch_size": null, "per_gpu_train_batch_size": null, "prediction_loss_only": false, "remove_unused_columns": true, "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "save_steps": 5000, "save_total_limit": 50, "seed": 17, "tpu_metrics_debug": false, "tpu_num_cores": null, "warmup_steps": 5400, "weight_decay": 0.0}, "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--6b3d26fc7262a898/checkpoint-110000"}, "f1": 88.96065210705885, "cat_fun_name": "is_new_xp", "annotate": "29"}}
{"fill_rate": 0.22240909529320985, "f1": 87.9662592155432, "meta": {"fill_rate": 0.22240909529320985, "speedup": 2.0527490991469155, "checkpoint": {"config": {"_name_or_path": "bert-base-uncased", "architectures": ["BertForQuestionAnswering"], "attention_probs_dropout_prob": 0.1, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-12, "max_position_embeddings": 512, "model_type": "bert", "num_attention_heads": 12, "num_hidden_layers": 12, "pad_token_id": 0, "position_embedding_type": "absolute", "type_vocab_size": 2, "vocab_size": 30522}, "eval_metrics": {"exact_match": 80.5771050141911, "f1": 87.9662592155432}, "sparse_args": {"ampere_pruning_method": "disabled", "attention_block_cols": 32, "attention_block_rows": 32, "attention_lambda": 1.0, "attention_output_with_dense": 0, "attention_pruning_method": "sigmoied_threshold", "bias_mask": true, "dense_block_cols": 1, "dense_block_rows": 1, "dense_lambda": 1.0, "dense_pruning_method": "sigmoied_threshold:1d_alt", "distil_alpha_ce": 0.1, "distil_alpha_teacher": 0.9, "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad", "distil_temperature": 2.0, "final_ampere_temperature": 20.0, "final_finetune": false, "final_threshold": 0.1, "final_warmup": 10, "initial_ampere_temperature": 0.0, "initial_threshold": 0, "initial_warmup": 1, "mask_init": "constant", "mask_scale": 0.0, "mask_scores_learning_rate": 0.01, "regularization": "l1", "regularization_final_lambda": 10}, "speed": {"cuda_eval_elapsed_time": 18.801320152282717, "eval_elapsed_time": 25.66945153940469}, "speedup": 2.0527490991469155, "stats": {"layers": {"0": {"linear_attention_nnz": 961536, "linear_attention_total": 2359296, "linear_dense_nnz": 543744, "linear_dense_total": 4718592, "linear_nnz": 1505280, "linear_total": 7077888, "nnz": 1511394, "total": 7087872}, "1": {"linear_attention_nnz": 749568, "linear_attention_total": 2359296, "linear_dense_nnz": 764928, "linear_dense_total": 4718592, "linear_nnz": 1514496, "linear_total": 7077888, "nnz": 1520466, "total": 7087872}, "10": {"linear_attention_nnz": 601088, "linear_attention_total": 2359296, "linear_dense_nnz": 176640, "linear_dense_total": 4718592, "linear_nnz": 777728, "linear_total": 7077888, "nnz": 783379, "total": 7087872}, "11": {"linear_attention_nnz": 449536, "linear_attention_total": 2359296, "linear_dense_nnz": 259584, "linear_dense_total": 4718592, "linear_nnz": 709120, "linear_total": 7077888, "nnz": 714729, "total": 7087872}, "2": {"linear_attention_nnz": 1303552, "linear_attention_total": 2359296, "linear_dense_nnz": 835584, "linear_dense_total": 4718592, "linear_nnz": 2139136, "linear_total": 7077888, "nnz": 2145856, "total": 7087872}, "3": {"linear_attention_nnz": 1416192, "linear_attention_total": 2359296, "linear_dense_nnz": 866304, "linear_dense_total": 4718592, "linear_nnz": 2282496, "linear_total": 7077888, "nnz": 2289300, "total": 7087872}, "4": {"linear_attention_nnz": 1542144, "linear_attention_total": 2359296, "linear_dense_nnz": 893952, "linear_dense_total": 4718592, "linear_nnz": 2436096, "linear_total": 7077888, "nnz": 2443110, "total": 7087872}, "5": {"linear_attention_nnz": 1292288, "linear_attention_total": 2359296, "linear_dense_nnz": 797184, "linear_dense_total": 4718592, "linear_nnz": 2089472, "linear_total": 7077888, "nnz": 2096263, "total": 7087872}, "6": {"linear_attention_nnz": 1261568, "linear_attention_total": 2359296, "linear_dense_nnz": 637440, "linear_dense_total": 4718592, "linear_nnz": 1899008, "linear_total": 7077888, "nnz": 1905759, "total": 7087872}, "7": {"linear_attention_nnz": 1064960, "linear_attention_total": 2359296, "linear_dense_nnz": 473088, "linear_dense_total": 4718592, "linear_nnz": 1538048, "linear_total": 7077888, "nnz": 1544276, "total": 7087872}, "8": {"linear_attention_nnz": 1018880, "linear_attention_total": 2359296, "linear_dense_nnz": 241152, "linear_dense_total": 4718592, "linear_nnz": 1260032, "linear_total": 7077888, "nnz": 1266205, "total": 7087872}, "9": {"linear_attention_nnz": 619520, "linear_attention_total": 2359296, "linear_dense_nnz": 119808, "linear_dense_total": 4718592, "linear_nnz": 739328, "linear_total": 7077888, "nnz": 744974, "total": 7087872}}, "linear_nnz": 18890240, "linear_sparsity": 77.75909047067901, "linear_total": 84934656, "nnz": 42804433, "pruned_heads": {"0": [0, 2, 4, 5, 6], "1": [0, 2, 3, 5, 6, 7, 8], "10": [1, 4, 5, 6, 7, 8], "11": [2, 5, 7, 8, 10, 11], "2": [8, 4, 7], "3": [2, 4, 6], "4": [1, 2], "5": [1, 2, 6], "6": [2, 3], "7": [1, 3, 6, 7, 11], "8": [0, 8, 3, 4], "9": [1, 4, 5, 7, 9, 10]}, "total": 108893186, "total_sparsity": 60.69135767595228}, "training_args": {"adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "dataloader_drop_last": false, "dataloader_num_workers": 0, "debug": false, "disable_tqdm": false, "do_eval": 1, "do_predict": false, "do_train": 1, "eval_accumulation_steps": null, "eval_steps": 5000, "evaluation_strategy": "steps", "fp16": false, "fp16_opt_level": "O1", "gradient_accumulation_steps": 1, "greater_is_better": null, "ignore_data_skip": false, "label_names": null, "learning_rate": 3e-05, "load_best_model_at_end": false, "local_rank": -1, "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad4/", "logging_first_step": false, "logging_steps": 250, "max_grad_norm": 1.0, "max_steps": -1, "metric_for_best_model": null, "model_parallel": false, "no_cuda": false, "num_train_epochs": 20, "optimize_model_before_eval": "disabled", "output_dir": "/data_2to/devel_data/nn_pruning/output/squad4/", "overwrite_output_dir": 1, "past_index": -1, "per_device_eval_batch_size": 8, "per_device_train_batch_size": 16, "per_gpu_eval_batch_size": null, "per_gpu_train_batch_size": null, "prediction_loss_only": false, "remove_unused_columns": true, "run_name": "/data_2to/devel_data/nn_pruning/output/squad4/", "save_steps": 5000, "save_total_limit": 50, "seed": 17, "tpu_metrics_debug": false, "tpu_num_cores": null, "warmup_steps": 5400, "weight_decay": 0.0}, "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--754f92d6579864ca/checkpoint-95000"}, "f1": 87.9662592155432, "cat_fun_name": "is_new_xp", "annotate": "22"}}
{"fill_rate": 0.15897472993827155, "f1": 86.63938864881486, "meta": {"fill_rate": 0.15897472993827155, "speedup": 2.3158516160525413, "checkpoint": {"config": {"_name_or_path": "bert-base-uncased", "architectures": ["BertForQuestionAnswering"], "attention_probs_dropout_prob": 0.1, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-12, "max_position_embeddings": 512, "model_type": "bert", "num_attention_heads": 12, "num_hidden_layers": 12, "pad_token_id": 0, "position_embedding_type": "absolute", "type_vocab_size": 2, "vocab_size": 30522}, "eval_metrics": {"exact_match": 78.77010406811732, "f1": 86.63938864881486}, "sparse_args": {"ampere_pruning_method": "disabled", "attention_block_cols": 32, "attention_block_rows": 32, "attention_lambda": 1.0, "attention_output_with_dense": 0, "attention_pruning_method": "sigmoied_threshold", "bias_mask": true, "dense_block_cols": 1, "dense_block_rows": 1, "dense_lambda": 1.0, "dense_pruning_method": "sigmoied_threshold:1d_alt", "distil_alpha_ce": 0.1, "distil_alpha_teacher": 0.9, "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad", "distil_temperature": 2.0, "final_ampere_temperature": 20.0, "final_finetune": false, "final_threshold": 0.1, "final_warmup": 10, "initial_ampere_temperature": 0.0, "initial_threshold": 0, "initial_warmup": 1, "mask_init": "constant", "mask_scale": 0.0, "mask_scores_learning_rate": 0.01, "regularization": "l1", "regularization_final_lambda": 20}, "speed": {"cuda_eval_elapsed_time": 16.665313415527343, "eval_elapsed_time": 23.629751751199365}, "speedup": 2.3158516160525413, "stats": {"layers": {"0": {"linear_attention_nnz": 677888, "linear_attention_total": 2359296, "linear_dense_nnz": 310272, "linear_dense_total": 4718592, "linear_nnz": 988160, "linear_total": 7077888, "nnz": 993834, "total": 7087872}, "1": {"linear_attention_nnz": 689152, "linear_attention_total": 2359296, "linear_dense_nnz": 436224, "linear_dense_total": 4718592, "linear_nnz": 1125376, "linear_total": 7077888, "nnz": 1131132, "total": 7087872}, "10": {"linear_attention_nnz": 434176, "linear_attention_total": 2359296, "linear_dense_nnz": 121344, "linear_dense_total": 4718592, "linear_nnz": 555520, "linear_total": 7077888, "nnz": 560943, "total": 7087872}, "11": {"linear_attention_nnz": 334848, "linear_attention_total": 2359296, "linear_dense_nnz": 156672, "linear_dense_total": 4718592, "linear_nnz": 491520, "linear_total": 7077888, "nnz": 496838, "total": 7087872}, "2": {"linear_attention_nnz": 1087488, "linear_attention_total": 2359296, "linear_dense_nnz": 543744, "linear_dense_total": 4718592, "linear_nnz": 1631232, "linear_total": 7077888, "nnz": 1637570, "total": 7087872}, "3": {"linear_attention_nnz": 1189888, "linear_attention_total": 2359296, "linear_dense_nnz": 565248, "linear_dense_total": 4718592, "linear_nnz": 1755136, "linear_total": 7077888, "nnz": 1761552, "total": 7087872}, "4": {"linear_attention_nnz": 1104896, "linear_attention_total": 2359296, "linear_dense_nnz": 589824, "linear_dense_total": 4718592, "linear_nnz": 1694720, "linear_total": 7077888, "nnz": 1701216, "total": 7087872}, "5": {"linear_attention_nnz": 818176, "linear_attention_total": 2359296, "linear_dense_nnz": 514560, "linear_dense_total": 4718592, "linear_nnz": 1332736, "linear_total": 7077888, "nnz": 1338767, "total": 7087872}, "6": {"linear_attention_nnz": 882688, "linear_attention_total": 2359296, "linear_dense_nnz": 442368, "linear_dense_total": 4718592, "linear_nnz": 1325056, "linear_total": 7077888, "nnz": 1331200, "total": 7087872}, "7": {"linear_attention_nnz": 846848, "linear_attention_total": 2359296, "linear_dense_nnz": 322560, "linear_dense_total": 4718592, "linear_nnz": 1169408, "linear_total": 7077888, "nnz": 1175442, "total": 7087872}, "8": {"linear_attention_nnz": 732160, "linear_attention_total": 2359296, "linear_dense_nnz": 167424, "linear_dense_total": 4718592, "linear_nnz": 899584, "linear_total": 7077888, "nnz": 905581, "total": 7087872}, "9": {"linear_attention_nnz": 449536, "linear_attention_total": 2359296, "linear_dense_nnz": 84480, "linear_dense_total": 4718592, "linear_nnz": 534016, "linear_total": 7077888, "nnz": 539287, "total": 7087872}}, "linear_nnz": 13502464, "linear_sparsity": 84.10252700617285, "linear_total": 84934656, "nnz": 37412084, "pruned_heads": {"0": [0, 2, 4, 5, 6, 7, 11], "1": [0, 2, 3, 5, 6, 7, 8], "10": [1, 2, 4, 5, 6, 7, 8], "11": [0, 2, 5, 7, 8, 10, 11], "2": [8, 4, 7], "3": [2, 4, 6, 7], "4": [1, 2, 11], "5": [1, 2, 5, 6, 7, 11], "6": [10, 2, 3, 7], "7": [1, 3, 6, 7, 11], "8": [0, 8, 3, 4], "9": [1, 4, 5, 7, 9, 10]}, "total": 108893186, "total_sparsity": 65.64331950026698}, "training_args": {"adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "dataloader_drop_last": false, "dataloader_num_workers": 0, "debug": false, "disable_tqdm": false, "do_eval": 1, "do_predict": false, "do_train": 1, "eval_accumulation_steps": null, "eval_steps": 5000, "evaluation_strategy": "steps", "fp16": false, "fp16_opt_level": "O1", "gradient_accumulation_steps": 1, "greater_is_better": null, "ignore_data_skip": false, "label_names": null, "learning_rate": 3e-05, "load_best_model_at_end": false, "local_rank": -1, "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad4/", "logging_first_step": false, "logging_steps": 250, "max_grad_norm": 1.0, "max_steps": -1, "metric_for_best_model": null, "model_parallel": false, "no_cuda": false, "num_train_epochs": 20, "optimize_model_before_eval": "disabled", "output_dir": "/data_2to/devel_data/nn_pruning/output/squad4/", "overwrite_output_dir": 1, "past_index": -1, "per_device_eval_batch_size": 8, "per_device_train_batch_size": 16, "per_gpu_eval_batch_size": null, "per_gpu_train_batch_size": null, "prediction_loss_only": false, "remove_unused_columns": true, "run_name": "/data_2to/devel_data/nn_pruning/output/squad4/", "save_steps": 5000, "save_total_limit": 50, "seed": 17, "tpu_metrics_debug": false, "tpu_num_cores": null, "warmup_steps": 5400, "weight_decay": 0.0}, "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad4___dpm-sigmoied_threshold:1d_alt_ap--17cd29ad8a563746/checkpoint-110000"}, "f1": 86.63938864881486, "cat_fun_name": "is_new_xp", "annotate": "15"}}
{"fill_rate": 0.1255726755401234, "f1": 85.80872913704097, "meta": {"fill_rate": 0.1255726755401234, "speedup": 2.6437372826229817, "checkpoint": {"config": {"_name_or_path": "bert-base-uncased", "architectures": ["BertForQuestionAnswering"], "attention_probs_dropout_prob": 0.1, "gradient_checkpointing": false, "hidden_act": "gelu", "hidden_dropout_prob": 0.1, "hidden_size": 768, "initializer_range": 0.02, "intermediate_size": 3072, "layer_norm_eps": 1e-12, "max_position_embeddings": 512, "model_type": "bert", "num_attention_heads": 12, "num_hidden_layers": 12, "pad_token_id": 0, "position_embedding_type": "absolute", "type_vocab_size": 2, "vocab_size": 30522}, "eval_metrics": {"exact_match": 77.67265846736045, "f1": 85.80872913704097}, "sparse_args": {"ampere_pruning_method": "disabled", "attention_block_cols": 32, "attention_block_rows": 32, "attention_lambda": 1.0, "attention_output_with_dense": 0, "attention_pruning_method": "sigmoied_threshold", "bias_mask": true, "dense_block_cols": 1, "dense_block_rows": 1, "dense_lambda": 1.0, "dense_pruning_method": "sigmoied_threshold:1d_alt", "distil_alpha_ce": 0.1, "distil_alpha_teacher": 0.9, "distil_teacher_name_or_path": "bert-large-uncased-whole-word-masking-finetuned-squad", "distil_temperature": 2.0, "final_ampere_temperature": 20.0, "final_finetune": false, "final_threshold": 0.1, "final_warmup": 10, "initial_ampere_temperature": 0.0, "initial_threshold": 0, "initial_warmup": 1, "mask_init": "constant", "mask_scale": 0.0, "mask_scores_learning_rate": 0.01, "regularization": "l1", "regularization_final_lambda": 30}, "speed": {"cuda_eval_elapsed_time": 14.598422187805175, "eval_elapsed_time": 21.40814032033086}, "speedup": 2.6437372826229817, "stats": {"layers": {"0": {"linear_attention_nnz": 528384, "linear_attention_total": 2359296, "linear_dense_nnz": 222720, "linear_dense_total": 4718592, "linear_nnz": 751104, "linear_total": 7077888, "nnz": 756497, "total": 7087872}, "1": {"linear_attention_nnz": 551936, "linear_attention_total": 2359296, "linear_dense_nnz": 344064, "linear_dense_total": 4718592, "linear_nnz": 896000, "linear_total": 7077888, "nnz": 901504, "total": 7087872}, "2": {"linear_attention_nnz": 867328, "linear_attention_total": 2359296, "linear_dense_nnz": 437760, "linear_dense_total": 4718592, "linear_nnz": 1305088, "linear_total": 7077888, "nnz": 1311069, "total": 7087872}, "3": {"linear_attention_nnz": 987136, "linear_attention_total": 2359296, "linear_dense_nnz": 440832, "linear_dense_total": 4718592, "linear_nnz": 1427968, "linear_total": 7077888, "nnz": 1434207, "total": 7087872}, "4": {"linear_attention_nnz": 702464, "linear_attention_total": 2359296, "linear_dense_nnz": 443904, "linear_dense_total": 4718592, "linear_nnz": 1146368, "linear_total": 7077888, "nnz": 1152257, "total": 7087872}, "5": {"linear_attention_nnz": 637952, "linear_attention_total": 2359296, "linear_dense_nnz": 427008, "linear_dense_total": 4718592, "linear_nnz": 1064960, "linear_total": 7077888, "nnz": 1070710, "total": 7087872}, "6": {"linear_attention_nnz": 703488, "linear_attention_total": 2359296, "linear_dense_nnz": 334848, "linear_dense_total": 4718592, "linear_nnz": 1038336, "linear_total": 7077888, "nnz": 1044218, "total": 7087872}, "7": {"linear_attention_nnz": 776192, "linear_attention_total": 2359296, "linear_dense_nnz": 265728, "linear_dense_total": 4718592, "linear_nnz": 1041920, "linear_total": 7077888, "nnz": 1047789, "total": 7087872}, "8": {"linear_attention_nnz": 508928, "linear_attention_total": 2359296, "linear_dense_nnz": 138240, "linear_dense_total": 4718592, "linear_nnz": 647168, "linear_total": 7077888, "nnz": 652730, "total": 7087872}, "9": {"linear_attention_nnz": 386048, "linear_attention_total": 2359296, "linear_dense_nnz": 67584, "linear_dense_total": 4718592, "linear_nnz": 453632, "linear_total": 7077888, "nnz": 458828, "total": 7087872}, "10": {"linear_attention_nnz": 399360, "linear_attention_total": 2359296, "linear_dense_nnz": 104448, "linear_dense_total": 4718592, "linear_nnz": 503808, "linear_total": 7077888, "nnz": 509252, "total": 7087872}, "11": {"linear_attention_nnz": 260096, "linear_attention_total": 2359296, "linear_dense_nnz": 129024, "linear_dense_total": 4718592, "linear_nnz": 389120, "linear_total": 7077888, "nnz": 394228, "total": 7087872}}, "linear_nnz": 10665472, "linear_sparsity": 87.44273244598766, "linear_total": 84934656, "nnz": 34572011, "pruned_heads": {"0": [0, 2, 4, 5, 6, 7, 9, 11], "1": [0, 2, 3, 5, 6, 7, 8, 9], "2": [8, 1, 4, 7], "3": [2, 4, 6, 7, 10], "4": [1, 2, 6, 7, 8, 11], "5": [0, 1, 2, 5, 6, 7, 11], "6": [0, 2, 3, 7, 10], "7": [1, 3, 6, 7, 11], "8": [0, 4, 5, 6, 7, 8], "9": [1, 2, 4, 5, 7, 9, 10], "10": [1, 2, 4, 5, 6, 7, 8], "11": [0, 2, 3, 5, 7, 8, 10, 11]}, "total": 108893186, "total_sparsity": 68.25144688116664}, "training_args": {"adam_beta1": 0.9, "adam_beta2": 0.999, "adam_epsilon": 1e-08, "dataloader_drop_last": false, "dataloader_num_workers": 0, "debug": false, "disable_tqdm": false, "do_eval": 1, "do_predict": false, "do_train": 1, "eval_accumulation_steps": null, "eval_steps": 5000, "evaluation_strategy": "steps", "fp16": false, "fp16_opt_level": "O1", "gradient_accumulation_steps": 1, "greater_is_better": null, "ignore_data_skip": false, "label_names": null, "learning_rate": 3e-05, "load_best_model_at_end": false, "local_rank": -1, "logging_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "logging_first_step": false, "logging_steps": 250, "max_grad_norm": 1.0, "max_steps": -1, "metric_for_best_model": null, "model_parallel": false, "no_cuda": false, "num_train_epochs": 20, "optimize_model_before_eval": "disabled", "output_dir": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "overwrite_output_dir": 1, "past_index": -1, "per_device_eval_batch_size": 8, "per_device_train_batch_size": 16, "per_gpu_eval_batch_size": null, "per_gpu_train_batch_size": null, "prediction_loss_only": false, "remove_unused_columns": true, "run_name": "/data_2to/devel_data/nn_pruning/output/squad_test4/", "save_steps": 5000, "save_total_limit": 50, "seed": 17, "tpu_metrics_debug": false, "tpu_num_cores": null, "warmup_steps": 5400, "weight_decay": 0.0}, "path": "/data_2to/devel_data/nn_pruning/output/squad_test4/hp_od-__data_2to__devel_data__nn_pruning__output__squad_test4___es-steps_nte20_ls250_stl50_est5000_rn-__data_2to__devel_data__nn_pruning__output__squad_test4___dpm-sigmoied_threshold--696f4785b3ba52e7/checkpoint-110000"}, "f1": 85.80872913704097, "cat_fun_name": "is_new_xp", "annotate": "12"}}
