Upload folder using huggingface_hub

.gitattributes

@@ -42,3 +42,39 @@ llmtf_eval/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
 llmtf_eval/russiannlp_rucola_custom.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/daru_treewayabstractive.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/darumeru_MultiQ.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/darumeru_USE.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/darumeru_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-10000/llmtf_eval/russiannlp_rucola_custom.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/daru_treewayabstractive.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/darumeru_MultiQ.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/darumeru_USE.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/darumeru_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-20000/llmtf_eval/russiannlp_rucola_custom.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/daru_treewayabstractive.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/darumeru_MultiQ.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/darumeru_USE.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/darumeru_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-30000/llmtf_eval/russiannlp_rucola_custom.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/daru_treewayabstractive.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/daru_treewayextractive.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/darumeru_MultiQ.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/darumeru_USE.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/darumeru_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/darumeru_ruTiE.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/nlpcoreteam_enMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/nlpcoreteam_ruMMLU.jsonl filter=lfs diff=lfs merge=lfs -text
+checkpoint-40000/llmtf_eval/russiannlp_rucola_custom.jsonl filter=lfs diff=lfs merge=lfs -text

checkpoint-10000/llmtf_eval/daru_treewayabstractive.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8275bb778c225adc5322d97fe7a0ee3e0cfe3d8b8fc693f8d25c2d304d1f8f4
+size 15397908

checkpoint-10000/llmtf_eval/daru_treewayabstractive_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 512,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 500,
+        "method": "generate"
+    }
+}

checkpoint-10000/llmtf_eval/daru_treewayabstractive_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "daru/treewayabstractive",
+    "results": {
+        "rouge1": 0.35130012760651375,
+        "rouge2": 0.14546804184711307
+    },
+    "leaderboard_result": 0.2483840847268134
+}

checkpoint-10000/llmtf_eval/daru_treewayextractive.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2bf400b388293d3bef971f24a452cb67d1a7e4479f9ca701990028a5d936c596
+size 204837133

checkpoint-10000/llmtf_eval/daru_treewayextractive_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 500,
+        "method": "calculate_logsoftmax"
+    }
+}

checkpoint-10000/llmtf_eval/daru_treewayextractive_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "daru/treewayextractive",
+    "results": {
+        "r-prec": 0.37500505050505045
+    },
+    "leaderboard_result": 0.37500505050505045
+}

checkpoint-10000/llmtf_eval/darumeru_MultiQ.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03e18f1af5bb05e8518010ff2b0bcc91c22c6ab6a08fb7feb267399fbaceb90b
+size 21499614

checkpoint-10000/llmtf_eval/darumeru_MultiQ_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_MultiQ_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/MultiQ",
+    "results": {
+        "f1": 0.6329772681111111,
+        "em": 0.5535372848948374
+    },
+    "leaderboard_result": 0.5932572765029742
+}

checkpoint-10000/llmtf_eval/darumeru_PARus_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_PARus_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/PARus",
+    "results": {
+        "acc": 0.49
+    },
+    "leaderboard_result": 0.49
+}

checkpoint-10000/llmtf_eval/darumeru_RCB_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_RCB_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/RCB",
+    "results": {
+        "acc": 0.36818181818181817,
+        "f1_macro": 0.2858134159609725
+    },
+    "leaderboard_result": 0.3269976170713953
+}

checkpoint-10000/llmtf_eval/darumeru_RWSD_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_RWSD_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/RWSD",
+    "results": {
+        "acc": 0.5588235294117647
+    },
+    "leaderboard_result": 0.5588235294117647
+}

checkpoint-10000/llmtf_eval/darumeru_USE.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9afa99139da49162ff319147cd84f6b6340baa1e04e05a1bd8daed42dbd5ae5f
+size 10588057

checkpoint-10000/llmtf_eval/darumeru_USE_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_USE_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/USE",
+    "results": {
+        "grade_norm": 0.11764705882352941
+    },
+    "leaderboard_result": 0.11764705882352941
+}

checkpoint-10000/llmtf_eval/darumeru_cp_para_en_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1024,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_cp_para_en_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_para_en",
+    "results": {
+        "symbol_per_token": 4.158508608710815,
+        "len": 0.9999382716049382,
+        "lcs": 0.9918474208388305
+    },
+    "leaderboard_result": 0.9918474208388305
+}

checkpoint-10000/llmtf_eval/darumeru_cp_para_ru_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 1024,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_cp_para_ru_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_para_ru",
+    "results": {
+        "symbol_per_token": 4.579997922038179,
+        "len": 0.98302719550087,
+        "lcs": 0.8871016370928306
+    },
+    "leaderboard_result": 0.8871016370928306
+}

checkpoint-10000/llmtf_eval/darumeru_cp_sent_en_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 128,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_cp_sent_en_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_sent_en",
+    "results": {
+        "symbol_per_token": 4.107729393968992,
+        "len": 0.999850948509485,
+        "lcs": 0.9985196838877463
+    },
+    "leaderboard_result": 0.999850948509485
+}

checkpoint-10000/llmtf_eval/darumeru_cp_sent_ru_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 128,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "generate"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_cp_sent_ru_total.jsonl

@@ -0,0 +1,9 @@
+{
+    "task_name": "darumeru/cp_sent_ru",
+    "results": {
+        "symbol_per_token": 4.304202137681612,
+        "len": 0.9876855195247346,
+        "lcs": 0.9616480132015096
+    },
+    "leaderboard_result": 0.9876855195247346
+}

checkpoint-10000/llmtf_eval/darumeru_ruMMLU.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e97cb89ce8573ad1d294b2a96231b4e0afeda4956f53b447af1071309cb3a24
+size 89330306

checkpoint-10000/llmtf_eval/darumeru_ruMMLU_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_ruMMLU_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/ruMMLU",
+    "results": {
+        "acc": 0.45754764042701784
+    },
+    "leaderboard_result": 0.45754764042701784
+}

checkpoint-10000/llmtf_eval/darumeru_ruOpenBookQA_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_ruOpenBookQA_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/ruOpenBookQA",
+    "results": {
+        "acc": 0.6451890034364262,
+        "f1_macro": 0.6419148808938866
+    },
+    "leaderboard_result": 0.6435519421651563
+}

checkpoint-10000/llmtf_eval/darumeru_ruTiE.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbf42740a9d0546b7353080a0fdc97fd8174dfc92004a587bb64f4f8e2e3b2c9
+size 14860285

checkpoint-10000/llmtf_eval/darumeru_ruTiE_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_ruTiE_total.jsonl

@@ -0,0 +1,7 @@
+{
+    "task_name": "darumeru/ruTiE",
+    "results": {
+        "acc": 0.26046511627906976
+    },
+    "leaderboard_result": 0.26046511627906976
+}

checkpoint-10000/llmtf_eval/darumeru_ruWorldTree_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}

checkpoint-10000/llmtf_eval/darumeru_ruWorldTree_total.jsonl

@@ -0,0 +1,8 @@
+{
+    "task_name": "darumeru/ruWorldTree",
+    "results": {
+        "acc": 0.8,
+        "f1_macro": 0.7954691734417344
+    },
+    "leaderboard_result": 0.7977345867208672
+}

checkpoint-10000/llmtf_eval/evaluation_log.txt

@@ -0,0 +1,273 @@
+INFO: 2024-07-17 13:32:27,653: llmtf.base.evaluator: Starting eval on ['darumeru/multiq', 'darumeru/parus', 'darumeru/rcb', 'darumeru/ruopenbookqa', 'darumeru/rutie', 'darumeru/ruworldtree', 'darumeru/rwsd', 'darumeru/use', 'russiannlp/rucola_custom']
+INFO: 2024-07-17 13:32:27,654: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:32:27,654: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:32:28,111: llmtf.base.evaluator: Starting eval on ['darumeru/rummlu']
+INFO: 2024-07-17 13:32:28,111: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:32:28,111: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:32:29,322: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/rummlu']
+INFO: 2024-07-17 13:32:29,322: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:32:29,322: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:32:31,580: llmtf.base.evaluator: Starting eval on ['nlpcoreteam/enmmlu']
+INFO: 2024-07-17 13:32:31,581: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:32:31,581: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:32:33,359: llmtf.base.evaluator: Starting eval on ['daru/treewayabstractive']
+INFO: 2024-07-17 13:32:33,359: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:32:33,359: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:32:35,256: llmtf.base.evaluator: Starting eval on ['daru/treewayextractive']
+INFO: 2024-07-17 13:32:35,257: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:32:35,257: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:32:37,783: llmtf.base.evaluator: Starting eval on ['darumeru/cp_sent_ru', 'darumeru/cp_sent_en', 'darumeru/cp_para_ru', 'darumeru/cp_para_en']
+INFO: 2024-07-17 13:32:37,783: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:32:37,783: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:32:42,930: llmtf.base.darumeru/cp_sent_ru: Loading Dataset: 5.15s
+INFO: 2024-07-17 13:32:49,665: llmtf.base.daru/treewayextractive: Loading Dataset: 14.41s
+INFO: 2024-07-17 13:32:57,763: llmtf.base.darumeru/MultiQ: Loading Dataset: 30.11s
+INFO: 2024-07-17 13:32:58,321: llmtf.base.daru/treewayabstractive: Loading Dataset: 24.96s
+INFO: 2024-07-17 13:34:16,841: llmtf.base.darumeru/ruMMLU: Loading Dataset: 108.73s
+INFO: 2024-07-17 13:35:24,502: llmtf.base.darumeru/cp_sent_ru: Processing Dataset: 161.56s
+INFO: 2024-07-17 13:35:24,506: llmtf.base.darumeru/cp_sent_ru: Results for darumeru/cp_sent_ru:
+INFO: 2024-07-17 13:35:24,510: llmtf.base.darumeru/cp_sent_ru: {'symbol_per_token': 4.304202137681612, 'len': 0.9876855195247346, 'lcs': 0.9616480132015096}
+INFO: 2024-07-17 13:35:24,513: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:35:24,513: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:35:28,858: llmtf.base.darumeru/cp_sent_en: Loading Dataset: 4.34s
+INFO: 2024-07-17 13:36:12,580: llmtf.base.nlpcoreteam/enMMLU: Loading Dataset: 221.00s
+INFO: 2024-07-17 13:36:41,977: llmtf.base.darumeru/MultiQ: Processing Dataset: 224.21s
+INFO: 2024-07-17 13:36:41,978: llmtf.base.darumeru/MultiQ: Results for darumeru/MultiQ:
+INFO: 2024-07-17 13:36:41,997: llmtf.base.darumeru/MultiQ: {'f1': 0.6329772681111111, 'em': 0.5535372848948374}
+INFO: 2024-07-17 13:36:42,011: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:36:42,012: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:36:45,062: llmtf.base.darumeru/PARus: Loading Dataset: 3.05s
+INFO: 2024-07-17 13:36:51,891: llmtf.base.nlpcoreteam/ruMMLU: Loading Dataset: 262.57s
+INFO: 2024-07-17 13:36:54,092: llmtf.base.darumeru/PARus: Processing Dataset: 9.03s
+INFO: 2024-07-17 13:36:54,094: llmtf.base.darumeru/PARus: Results for darumeru/PARus:
+INFO: 2024-07-17 13:36:54,105: llmtf.base.darumeru/PARus: {'acc': 0.49}
+INFO: 2024-07-17 13:36:54,107: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:36:54,107: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:36:57,490: llmtf.base.darumeru/RCB: Loading Dataset: 3.38s
+INFO: 2024-07-17 13:37:02,868: llmtf.base.daru/treewayextractive: Processing Dataset: 253.19s
+INFO: 2024-07-17 13:37:02,882: llmtf.base.daru/treewayextractive: Results for daru/treewayextractive:
+INFO: 2024-07-17 13:37:03,092: llmtf.base.daru/treewayextractive: {'r-prec': 0.37500505050505045}
+INFO: 2024-07-17 13:37:03,133: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-17 13:37:03,139: llmtf.base.evaluator: 
+mean	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/cp_sent_ru
+0.611	0.375	0.593	0.490	0.988
+INFO: 2024-07-17 13:37:12,225: llmtf.base.darumeru/RCB: Processing Dataset: 14.72s
+INFO: 2024-07-17 13:37:12,240: llmtf.base.darumeru/RCB: Results for darumeru/RCB:
+INFO: 2024-07-17 13:37:12,246: llmtf.base.darumeru/RCB: {'acc': 0.36818181818181817, 'f1_macro': 0.2858134159609725}
+INFO: 2024-07-17 13:37:12,248: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:37:12,248: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:37:25,476: llmtf.base.darumeru/ruOpenBookQA: Loading Dataset: 13.23s
+INFO: 2024-07-17 13:38:29,729: llmtf.base.darumeru/cp_sent_en: Processing Dataset: 180.87s
+INFO: 2024-07-17 13:38:29,732: llmtf.base.darumeru/cp_sent_en: Results for darumeru/cp_sent_en:
+INFO: 2024-07-17 13:38:29,737: llmtf.base.darumeru/cp_sent_en: {'symbol_per_token': 4.107729393968992, 'len': 0.999850948509485, 'lcs': 0.9985196838877463}
+INFO: 2024-07-17 13:38:29,739: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:38:29,739: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:38:33,560: llmtf.base.darumeru/cp_para_ru: Loading Dataset: 3.82s
+INFO: 2024-07-17 13:38:59,037: llmtf.base.darumeru/ruOpenBookQA: Processing Dataset: 93.56s
+INFO: 2024-07-17 13:38:59,054: llmtf.base.darumeru/ruOpenBookQA: Results for darumeru/ruOpenBookQA:
+INFO: 2024-07-17 13:38:59,067: llmtf.base.darumeru/ruOpenBookQA: {'acc': 0.6451890034364262, 'f1_macro': 0.6419148808938866}
+INFO: 2024-07-17 13:38:59,083: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:38:59,083: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:39:06,305: llmtf.base.darumeru/ruTiE: Loading Dataset: 7.22s
+INFO: 2024-07-17 13:39:18,303: llmtf.base.daru/treewayabstractive: Processing Dataset: 379.98s
+INFO: 2024-07-17 13:39:18,306: llmtf.base.daru/treewayabstractive: Results for daru/treewayabstractive:
+INFO: 2024-07-17 13:39:18,323: llmtf.base.daru/treewayabstractive: {'rouge1': 0.35130012760651375, 'rouge2': 0.14546804184711307}
+INFO: 2024-07-17 13:39:18,331: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-17 13:39:18,398: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruOpenBookQA
+0.583	0.248	0.375	0.593	0.490	0.327	1.000	0.988	0.644
+INFO: 2024-07-17 13:42:57,147: llmtf.base.darumeru/cp_para_ru: Processing Dataset: 263.59s
+INFO: 2024-07-17 13:42:57,151: llmtf.base.darumeru/cp_para_ru: Results for darumeru/cp_para_ru:
+INFO: 2024-07-17 13:42:57,169: llmtf.base.darumeru/cp_para_ru: {'symbol_per_token': 4.579997922038179, 'len': 0.98302719550087, 'lcs': 0.8871016370928306}
+INFO: 2024-07-17 13:42:57,172: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:42:57,172: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:43:01,969: llmtf.base.darumeru/cp_para_en: Loading Dataset: 4.80s
+INFO: 2024-07-17 13:43:30,238: llmtf.base.darumeru/ruTiE: Processing Dataset: 263.93s
+INFO: 2024-07-17 13:43:30,240: llmtf.base.darumeru/ruTiE: Results for darumeru/ruTiE:
+INFO: 2024-07-17 13:43:30,283: llmtf.base.darumeru/ruTiE: {'acc': 0.26046511627906976}
+INFO: 2024-07-17 13:43:30,286: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:43:30,286: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:43:32,987: llmtf.base.darumeru/ruWorldTree: Loading Dataset: 2.70s
+INFO: 2024-07-17 13:43:38,701: llmtf.base.darumeru/ruWorldTree: Processing Dataset: 5.71s
+INFO: 2024-07-17 13:43:38,702: llmtf.base.darumeru/ruWorldTree: Results for darumeru/ruWorldTree:
+INFO: 2024-07-17 13:43:38,708: llmtf.base.darumeru/ruWorldTree: {'acc': 0.8, 'f1_macro': 0.7954691734417344}
+INFO: 2024-07-17 13:43:38,709: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:43:38,709: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:43:42,527: llmtf.base.darumeru/RWSD: Loading Dataset: 3.82s
+INFO: 2024-07-17 13:43:56,606: llmtf.base.darumeru/RWSD: Processing Dataset: 14.08s
+INFO: 2024-07-17 13:43:56,608: llmtf.base.darumeru/RWSD: Results for darumeru/RWSD:
+INFO: 2024-07-17 13:43:56,611: llmtf.base.darumeru/RWSD: {'acc': 0.5588235294117647}
+INFO: 2024-07-17 13:43:56,613: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:43:56,613: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:44:10,978: llmtf.base.darumeru/USE: Loading Dataset: 14.36s
+INFO: 2024-07-17 13:47:01,084: llmtf.base.darumeru/USE: Processing Dataset: 170.11s
+INFO: 2024-07-17 13:47:01,087: llmtf.base.darumeru/USE: Results for darumeru/USE:
+INFO: 2024-07-17 13:47:01,092: llmtf.base.darumeru/USE: {'grade_norm': 0.11764705882352941}
+INFO: 2024-07-17 13:47:01,099: llmtf.base.hfmodel: Updated generation_config.eos_token_id: [128001, 13]
+INFO: 2024-07-17 13:47:01,099: llmtf.base.hfmodel: Updated generation_config.stop_strings: ['\n', '\n\n']
+INFO: 2024-07-17 13:47:20,063: llmtf.base.russiannlp/rucola_custom: Loading Dataset: 18.96s
+INFO: 2024-07-17 13:47:45,782: llmtf.base.darumeru/cp_para_en: Processing Dataset: 283.80s
+INFO: 2024-07-17 13:47:45,785: llmtf.base.darumeru/cp_para_en: Results for darumeru/cp_para_en:
+INFO: 2024-07-17 13:47:45,803: llmtf.base.darumeru/cp_para_en: {'symbol_per_token': 4.158508608710815, 'len': 0.9999382716049382, 'lcs': 0.9918474208388305}
+INFO: 2024-07-17 13:47:45,806: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-17 13:47:45,846: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree
+0.591	0.248	0.375	0.593	0.490	0.327	0.559	0.118	0.992	0.887	1.000	0.988	0.644	0.260	0.798
+INFO: 2024-07-17 13:49:25,518: llmtf.base.russiannlp/rucola_custom: Processing Dataset: 125.44s
+INFO: 2024-07-17 13:49:25,522: llmtf.base.russiannlp/rucola_custom: Results for russiannlp/rucola_custom:
+INFO: 2024-07-17 13:49:25,534: llmtf.base.russiannlp/rucola_custom: {'acc': 0.6006458557588805, 'mcc': 0.3099732553523234}
+INFO: 2024-07-17 13:49:25,545: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-17 13:49:25,557: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	russiannlp/rucola_custom
+0.582	0.248	0.375	0.593	0.490	0.327	0.559	0.118	0.992	0.887	1.000	0.988	0.644	0.260	0.798	0.455
+INFO: 2024-07-17 13:49:35,861: llmtf.base.darumeru/ruMMLU: Processing Dataset: 919.02s
+INFO: 2024-07-17 13:49:35,863: llmtf.base.darumeru/ruMMLU: Results for darumeru/ruMMLU:
+INFO: 2024-07-17 13:49:35,872: llmtf.base.darumeru/ruMMLU: {'acc': 0.45754764042701784}
+INFO: 2024-07-17 13:49:35,956: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-17 13:49:35,969: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	russiannlp/rucola_custom
+0.574	0.248	0.375	0.593	0.490	0.327	0.559	0.118	0.992	0.887	1.000	0.988	0.458	0.644	0.260	0.798	0.455
+INFO: 2024-07-17 13:54:32,013: llmtf.base.nlpcoreteam/enMMLU: Processing Dataset: 1099.43s
+INFO: 2024-07-17 13:54:32,016: llmtf.base.nlpcoreteam/enMMLU: Results for nlpcoreteam/enMMLU:
+INFO: 2024-07-17 13:54:32,062: llmtf.base.nlpcoreteam/enMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.330000
+anatomy                              0.607407
+astronomy                            0.644737
+business_ethics                      0.590000
+clinical_knowledge                   0.683019
+college_biology                      0.701389
+college_chemistry                    0.430000
+college_computer_science             0.490000
+college_mathematics                  0.360000
+college_medicine                     0.601156
+college_physics                      0.401961
+computer_security                    0.760000
+conceptual_physics                   0.502128
+econometrics                         0.438596
+electrical_engineering               0.606897
+elementary_mathematics               0.378307
+formal_logic                         0.396825
+global_facts                         0.360000
+high_school_biology                  0.754839
+high_school_chemistry                0.487685
+high_school_computer_science         0.670000
+high_school_european_history         0.721212
+high_school_geography                0.792929
+high_school_government_and_politics  0.865285
+high_school_macroeconomics           0.610256
+high_school_mathematics              0.303704
+high_school_microeconomics           0.672269
+high_school_physics                  0.370861
+high_school_psychology               0.794495
+high_school_statistics               0.490741
+high_school_us_history               0.745098
+high_school_world_history            0.835443
+human_aging                          0.659193
+human_sexuality                      0.717557
+international_law                    0.793388
+jurisprudence                        0.740741
+logical_fallacies                    0.687117
+machine_learning                     0.508929
+management                           0.815534
+marketing                            0.833333
+medical_genetics                     0.710000
+miscellaneous                        0.757344
+moral_disputes                       0.656069
+moral_scenarios                      0.402235
+nutrition                            0.673203
+philosophy                           0.726688
+prehistory                           0.703704
+professional_accounting              0.475177
+professional_law                     0.453064
+professional_medicine                0.613971
+professional_psychology              0.656863
+public_relations                     0.672727
+security_studies                     0.755102
+sociology                            0.830846
+us_foreign_policy                    0.820000
+virology                             0.530120
+world_religions                      0.760234
+INFO: 2024-07-17 13:54:32,070: llmtf.base.nlpcoreteam/enMMLU:                                    metric
+subject                                  
+STEM                             0.510676
+humanities                       0.663217
+other (business, health, misc.)  0.636390
+social sciences                  0.718911
+INFO: 2024-07-17 13:54:32,092: llmtf.base.nlpcoreteam/enMMLU: {'acc': 0.6322983719525322}
+INFO: 2024-07-17 13:54:32,162: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-17 13:54:32,176: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	russiannlp/rucola_custom
+0.578	0.248	0.375	0.593	0.490	0.327	0.559	0.118	0.992	0.887	1.000	0.988	0.458	0.644	0.260	0.798	0.632	0.455
+INFO: 2024-07-17 13:55:12,888: llmtf.base.nlpcoreteam/ruMMLU: Processing Dataset: 1101.00s
+INFO: 2024-07-17 13:55:12,905: llmtf.base.nlpcoreteam/ruMMLU: Results for nlpcoreteam/ruMMLU:
+INFO: 2024-07-17 13:55:12,950: llmtf.base.nlpcoreteam/ruMMLU:                                        metric
+subject                                      
+abstract_algebra                     0.280000
+anatomy                              0.496296
+astronomy                            0.598684
+business_ethics                      0.530000
+clinical_knowledge                   0.509434
+college_biology                      0.527778
+college_chemistry                    0.350000
+college_computer_science             0.440000
+college_mathematics                  0.400000
+college_medicine                     0.502890
+college_physics                      0.294118
+computer_security                    0.620000
+conceptual_physics                   0.455319
+econometrics                         0.324561
+electrical_engineering               0.572414
+elementary_mathematics               0.354497
+formal_logic                         0.341270
+global_facts                         0.330000
+high_school_biology                  0.622581
+high_school_chemistry                0.448276
+high_school_computer_science         0.600000
+high_school_european_history         0.703030
+high_school_geography                0.606061
+high_school_government_and_politics  0.642487
+high_school_macroeconomics           0.530769
+high_school_mathematics              0.296296
+high_school_microeconomics           0.491597
+high_school_physics                  0.337748
+high_school_psychology               0.636697
+high_school_statistics               0.407407
+high_school_us_history               0.617647
+high_school_world_history            0.725738
+human_aging                          0.515695
+human_sexuality                      0.595420
+international_law                    0.752066
+jurisprudence                        0.601852
+logical_fallacies                    0.533742
+machine_learning                     0.455357
+management                           0.689320
+marketing                            0.705128
+medical_genetics                     0.560000
+miscellaneous                        0.578544
+moral_disputes                       0.575145
+moral_scenarios                      0.280447
+nutrition                            0.575163
+philosophy                           0.591640
+prehistory                           0.567901
+professional_accounting              0.354610
+professional_law                     0.370274
+professional_medicine                0.426471
+professional_psychology              0.490196
+public_relations                     0.527273
+security_studies                     0.697959
+sociology                            0.696517
+us_foreign_policy                    0.710000
+virology                             0.457831
+world_religions                      0.649123
+INFO: 2024-07-17 13:55:12,958: llmtf.base.nlpcoreteam/ruMMLU:                                    metric
+subject                                  
+STEM                             0.447804
+humanities                       0.562298
+other (business, health, misc.)  0.516527
+social sciences                  0.579128
+INFO: 2024-07-17 13:55:12,965: llmtf.base.nlpcoreteam/ruMMLU: {'acc': 0.5264394464387596}
+INFO: 2024-07-17 13:55:13,046: llmtf.base.evaluator: Ended eval
+INFO: 2024-07-17 13:55:13,060: llmtf.base.evaluator: 
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.575	0.248	0.375	0.593	0.490	0.327	0.559	0.118	0.992	0.887	1.000	0.988	0.458	0.644	0.260	0.798	0.632	0.526	0.455

checkpoint-10000/llmtf_eval/evaluation_results.txt

@@ -0,0 +1,2 @@
+mean	daru/treewayabstractive	daru/treewayextractive	darumeru/MultiQ	darumeru/PARus	darumeru/RCB	darumeru/RWSD	darumeru/USE	darumeru/cp_para_en	darumeru/cp_para_ru	darumeru/cp_sent_en	darumeru/cp_sent_ru	darumeru/ruMMLU	darumeru/ruOpenBookQA	darumeru/ruTiE	darumeru/ruWorldTree	nlpcoreteam/enMMLU	nlpcoreteam/ruMMLU	russiannlp/rucola_custom
+0.575	0.248	0.375	0.593	0.490	0.327	0.559	0.118	0.992	0.887	1.000	0.988	0.458	0.644	0.260	0.798	0.632	0.526	0.455

checkpoint-10000/llmtf_eval/nlpcoreteam_enMMLU.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bb596e0e05be6090ccc500e4c541224e4fc1a6e2a46f72b731187c85ed8880f7
+size 73142280

checkpoint-10000/llmtf_eval/nlpcoreteam_enMMLU_params.jsonl

@@ -0,0 +1,62 @@
+{
+    "custom_generation_config": null,
+    "model_params": {
+        "model_name_or_path": "/scratch/tikhomirov/workdir/data/models/llama3_darulm_20_05_24_part1-2_128000_unigram_full_lr2e4_bs256/checkpoint-10000",
+        "generation_config": {
+            "bos_token_id": 128000,
+            "do_sample": true,
+            "eos_token_id": [
+                128001,
+                13
+            ],
+            "max_length": 8192,
+            "max_new_tokens": 64,
+            "pad_token_id": 128001,
+            "stop_strings": [
+                "\n",
+                "\n\n"
+            ],
+            "temperature": 0.1,
+            "top_k": 40,
+            "top_p": 0.9,
+            "transformers_version": "4.38.2",
+            "trust_remote_code": [
+                false
+            ]
+        },
+        "conversation_template": {
+            "system_message_template": "{content}\n",
+            "user_message_template": "{content}\n",
+            "bot_message_template": "{content}\n\n",
+            "bot_message_template_incomplete": "{content}",
+            "user_role": "user",
+            "bot_role": "bot",
+            "system_role": "system",
+            "suffix": "",
+            "add_special_tokens": false,
+            "eos_token": [
+                "\n",
+                "\n\n"
+            ],
+            "global_prefix": "<|begin_of_text|>"
+        },
+        "load_in_8bit": false,
+        "torch_dtype": "auto",
+        "use_flash_attention_2": true,
+        "device_map": "cuda:0",
+        "use_fast_tokenizer": true,
+        "leading_space": true,
+        "space_token": 261,
+        "trust_remote_code": [
+            false
+        ],
+        "max_model_len": 8192
+    },
+    "task_params": {
+        "max_len": 4000,
+        "few_shot_count": 5,
+        "batch_size": 8,
+        "max_sample_per_dataset": 10000000000000,
+        "method": "calculate_tokens_proba"
+    }
+}