Adding evaluation results

evaluations/ar/AceGPT-v2-32B-Chat/acva_5_shot.json

@@ -0,0 +1,125 @@
+{
+  "results": {
+    "acva": {
+      "alias": "acva",
+      "acc,none": 0.7274397244546499,
+      "acc_stderr,none": 0.004771397968508457,
+      "acc_norm,none": 0.7157290470723306,
+      "acc_norm_stderr,none": 0.004833440968499389
+    }
+  },
+  "group_subtasks": {
+    "acva": []
+  },
+  "configs": {
+    "acva": {
+      "task": "acva",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _format_subject(subject):\n        \n        arabic_words = subtasks_ar[subtasks.index(subject)]\n        return arabic_words\n    \n    def _generate_subject(doc):\n        subject = _format_subject(doc[\"id\"].split(\"-\")[0])\n\n        return subject\n        \n    def _process_docs(doc):\n        keys = [\"\u0635\u062d\",\n                 \"\u062e\u0637\u0623\"]\n        subject = _generate_subject(doc)\n        gold = keys.index(doc['answer'])\n        out_doc = {\n            \"id\": doc[\"id\"],\n            \"query\":  \"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" + doc[\"question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\",\n            \"choices\": keys,\n            \"gold\": gold,\n            \"subject\": subject,\n        }\n        \n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d' \u0623\u0648 '\u062e\u0637\u0623' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "acva": 1.0
+  },
+  "n-shot": {
+    "acva": 5
+  },
+  "higher_is_better": {
+    "acva": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "acva": {
+      "original": 8710,
+      "effective": 8710
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1737779797.3395095,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.1",
+  "upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 26647.534977248,
+  "end_time": 27360.084961217,
+  "total_evaluation_time_seconds": "712.5499839689983"
+}

evaluations/ar/AceGPT-v2-32B-Chat/ar_ifeval_0_shot.json

@@ -0,0 +1,142 @@
+{
+  "results": {
+    "ar_ifeval": {
+      "alias": "ar_ifeval",
+      "prompt_level_strict_acc,none": 0.2574626865671642,
+      "prompt_level_strict_acc_stderr,none": 0.018903377119672635,
+      "inst_level_strict_acc,none": 0.6341296928327645,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.31529850746268656,
+      "prompt_level_loose_acc_stderr,none": 0.020087907677710036,
+      "inst_level_loose_acc,none": 0.6764505119453925,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "ar_ifeval": []
+  },
+  "configs": {
+    "ar_ifeval": {
+      "task": "ar_ifeval",
+      "dataset_path": "lm_eval/tasks/ar_ifeval/ar_ifeval.py",
+      "dataset_name": "ar_ifeval",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "process_results": "def process_results(doc, results):\n\n    response = results[0]\n    out_strict = process_sample(doc, response, 'strict')\n    out_loose = process_sample(doc, response, 'loose')\n\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0
+      }
+    }
+  },
+  "versions": {
+    "ar_ifeval": 4.0
+  },
+  "n-shot": {
+    "ar_ifeval": 0
+  },
+  "higher_is_better": {
+    "ar_ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ar_ifeval": {
+      "original": 536,
+      "effective": 536
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738794647.2071357,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {
+    "ar_ifeval": "d0b91e989c8b697090db63bf498d8e2d8dd80815a595e5f22845a8425bff22fa"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template_sha": "af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db",
+  "start_time": 1753623.131321269,
+  "end_time": 1761093.682009075,
+  "total_evaluation_time_seconds": "7470.550687805982"
+}

evaluations/ar/AceGPT-v2-32B-Chat/araMath_v3_5_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "araMath_v3": {
+      "alias": "araMath_v3",
+      "acc,none": 0.6446280991735537,
+      "acc_stderr,none": 0.019475010007284948,
+      "acc_norm,none": 0.6446280991735537,
+      "acc_norm_stderr,none": 0.019475010007284948
+    }
+  },
+  "group_subtasks": {
+    "araMath_v3": []
+  },
+  "configs": {
+    "araMath_v3": {
+      "task": "araMath_v3",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araMath_v3/araMath_v3.py",
+      "dataset_name": "araMath_v3",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def remove_prefix(choice):\n            prefixes = [\"(A)\", \"(B)\", \"(C)\", \"(D)\"]\n            for prefix in prefixes:\n                if choice.startswith(prefix + \" \"):\n                    return choice[len(prefix) + 1:]  \n            return choice \n\n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"options\"])]\n            )\n\n            prompt = f\"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_en.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 \u0628\u064a\u0646 'A\u060c B\u060c C\u060c D' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "araMath_v3": 0.0
+  },
+  "n-shot": {
+    "araMath_v3": 5
+  },
+  "higher_is_better": {
+    "araMath_v3": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araMath_v3": {
+      "original": 605,
+      "effective": 605
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738805225.8162587,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {
+    "araMath_v3": "17b2596f46d709ea107ed20bef044ca126de23a8e9bbc8ba0a9beef94fbc032d"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template_sha": "af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db",
+  "start_time": 1764201.606664753,
+  "end_time": 1764270.091855178,
+  "total_evaluation_time_seconds": "68.48519042483531"
+}

evaluations/ar/AceGPT-v2-32B-Chat/araPro_0_shot.json

@@ -0,0 +1,130 @@
+{
+  "results": {
+    "araPro": {
+      "alias": "araPro",
+      "acc,none": 0.671865626874625,
+      "acc_stderr,none": 0.006640213946839424,
+      "acc_norm,none": 0.671865626874625,
+      "acc_norm_stderr,none": 0.006640213946839424
+    }
+  },
+  "group_subtasks": {
+    "araPro": []
+  },
+  "configs": {
+    "araPro": {
+      "task": "araPro",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araPro/araPro.py",
+      "dataset_name": "araPro",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.replace('.', '') if '.' in choice[:2] else choice\n        \n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            \n            choice_num = ['choice1', 'choice2', 'choice3', 'choice4']\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(doc[choice_num[index]])}\\n\" for index, key in enumerate(keys)]\n            )\n\n            prompt = f\"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        #keys = [\"1\", \"2\", \"3\", \"4\"]\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\":  format_example(doc, keys), \n            \"choices\": keys,\n            \"gold\": doc[\"answer\"]-1,\n        }        \n\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question",
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "araPro": 2.0
+  },
+  "n-shot": {
+    "araPro": 0
+  },
+  "higher_is_better": {
+    "araPro": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araPro": {
+      "original": 5001,
+      "effective": 5001
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738802810.5474553,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {
+    "araPro": "2f706897ad0129e016cc8d6907f8bb4359c32403fc2d1b0a4e78717f424793da"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template_sha": "af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db",
+  "start_time": 1761786.552693387,
+  "end_time": 1761894.218775138,
+  "total_evaluation_time_seconds": "107.66608175099827"
+}

evaluations/ar/AceGPT-v2-32B-Chat/etec_v2_0_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "etec_v2": {
+      "alias": "etec_v2",
+      "acc,none": 0.6481187069422364,
+      "acc_stderr,none": 0.010996501146375258,
+      "acc_norm,none": 0.6481187069422364,
+      "acc_norm_stderr,none": 0.010996501146375258
+    }
+  },
+  "group_subtasks": {
+    "etec_v2": []
+  },
+  "configs": {
+    "etec_v2": {
+      "task": "etec_v2",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/etec_v2/etec.py",
+      "dataset_name": "etec_v2",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n        print(doc[\"label\"])\n        keys_ar = [\"\u0623\", \"\u0628\", \"\u062c\", \"\u062f\"]\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n                \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": int(doc[\"label\"])-1,\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d\n ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "etec_v2": 0.0
+  },
+  "n-shot": {
+    "etec_v2": 0
+  },
+  "higher_is_better": {
+    "etec_v2": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "etec_v2": {
+      "original": 1887,
+      "effective": 1887
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738805984.3189015,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {
+    "etec_v2": "697b8bfc7d6b0f85165e5cca6953182b09b7a2b0d79fa31e74cc3897f432de41"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template_sha": "af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db",
+  "start_time": 1764960.166542801,
+  "end_time": 1765035.801506021,
+  "total_evaluation_time_seconds": "75.63496321998537"
+}

evaluations/ar/AceGPT-v2-32B-Chat/exams_ar_5_shot.json

@@ -0,0 +1,127 @@
+{
+  "results": {
+    "exams_ar": {
+      "alias": "exams_ar",
+      "acc,none": 0.553072625698324,
+      "acc_stderr,none": 0.021474702941383872,
+      "acc_norm,none": 0.553072625698324,
+      "acc_norm_stderr,none": 0.021474702941383872
+    }
+  },
+  "group_subtasks": {
+    "exams_ar": []
+  },
+  "configs": {
+    "exams_ar": {
+      "task": "exams_ar",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/exams_ar",
+      "dataset_name": "exams_ar",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            \"\"\"\n            <prompt>\n            \u0633\u0624\u0627\u0644:\n            A. <choice1>\n            B. <choice2>\n            C. <choice3>\n            D. <choice4>\n            \u0627\u062c\u0627\u0628\u0629:\n            \"\"\"\n            \n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        def _format_subject(subject):\n            arabic_words = subtasks_ar[subtasks.index(subject)]\n            return arabic_words\n\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n    \n        subject = doc['id'].split(\"-\")[0]\n        description = f\"\ufed2\ufef4\ufee3\ufe8d \ufef2\ufee0\ufef3 \ufe84\ufeb4\ufe8c\ufedf\ufe93 \ufe8d\ufefc\ufea8\ufe98\ufef3\ufe8d\ufead \ufee2\ufee7 \ufee2\ufe98\ufecb\ufea9\ufea9 (\ufee2\ufecb \ufe8d\ufefa\ufe9f\ufe8e\ufe91\ufe8e\ufe97) \ufea1\ufeee\ufedf {_format_subject(subject)} \\n\" #\ufee2\ufee7 \ufed2\ufec0\ufee0\ufedb \ufe8e\ufea8\ufe97\ufead \ufe88\ufe9f\ufe8e\ufe91\ufe93 \ufeed\ufe8e\ufea3\ufea9\ufe93 \ufee2\ufee7 \ufe90\ufef4\ufee7 'A\u060c B\u060c C\u060c D' \ufea9\ufeee\ufee7 \ufeb5\ufeae\ufea3\\n\"\n\n        out_doc = {\n            \"idx\": doc[\"idx\"],\n            \"id\": doc[\"id\"],\n            'dsecription': description,\n            \"query\":  format_example(doc, keys), # \"Question: \" + doc[\"question\"]['stem'] + \"\\nAnswer:\",\n            \"choices\": keys,\n            \"gold\": [\"A\", \"B\", \"C\", \"D\"].index(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "description",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "exams_ar": 1.0
+  },
+  "n-shot": {
+    "exams_ar": 5
+  },
+  "higher_is_better": {
+    "exams_ar": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "exams_ar": {
+      "original": 537,
+      "effective": 537
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": "auto",
+    "batch_sizes": [
+      16
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1737780545.20475,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.1",
+  "upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 27395.295045238,
+  "end_time": 27506.949709817,
+  "total_evaluation_time_seconds": "111.65466457900038"
+}

evaluations/ar/AceGPT-v2-32B-Chat/gat_0_shot.json

@@ -0,0 +1,543 @@
+{
+  "results": {
+    "gat": {
+      "acc,none": 0.4321459927254484,
+      "acc_stderr,none": 0.0038347299693873033,
+      "alias": "gat"
+    },
+    "gat_algebra": {
+      "alias": " - gat_algebra",
+      "acc,none": 0.3992578849721707,
+      "acc_stderr,none": 0.009435653731651068
+    },
+    "gat_analogy": {
+      "alias": " - gat_analogy",
+      "acc,none": 0.2867030965391621,
+      "acc_stderr,none": 0.00863295163043938
+    },
+    "gat_arithmetic": {
+      "alias": " - gat_arithmetic",
+      "acc,none": 0.3894000736105999,
+      "acc_stderr,none": 0.009356458715331561
+    },
+    "gat_association": {
+      "alias": " - gat_association",
+      "acc,none": 0.4143540669856459,
+      "acc_stderr,none": 0.01524590184737997
+    },
+    "gat_comparisons": {
+      "alias": " - gat_comparisons",
+      "acc,none": 0.34672131147540985,
+      "acc_stderr,none": 0.013631312083187472
+    },
+    "gat_completion": {
+      "alias": " - gat_completion",
+      "acc,none": 0.5793388429752067,
+      "acc_stderr,none": 0.014197745251253151
+    },
+    "gat_contextual": {
+      "alias": " - gat_contextual",
+      "acc,none": 0.522239263803681,
+      "acc_stderr,none": 0.013837823280527494
+    },
+    "gat_geometry": {
+      "alias": " - gat_geometry",
+      "acc,none": 0.5013698630136987,
+      "acc_stderr,none": 0.026207022561245137
+    },
+    "gat_reading": {
+      "alias": " - gat_reading",
+      "acc,none": 0.585633270321361,
+      "acc_stderr,none": 0.009580200187530542
+    }
+  },
+  "groups": {
+    "gat": {
+      "acc,none": 0.4321459927254484,
+      "acc_stderr,none": 0.0038347299693873033,
+      "alias": "gat"
+    }
+  },
+  "group_subtasks": {
+    "gat": [
+      "gat_analogy",
+      "gat_association",
+      "gat_completion",
+      "gat_reading",
+      "gat_algebra",
+      "gat_arithmetic",
+      "gat_comparisons",
+      "gat_contextual",
+      "gat_geometry"
+    ]
+  },
+  "configs": {
+    "gat_algebra": {
+      "task": "gat_algebra",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_analogy": {
+      "task": "gat_analogy",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "analogy",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_arithmetic": {
+      "task": "gat_arithmetic",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "arithmetic",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_association": {
+      "task": "gat_association",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "association",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_comparisons": {
+      "task": "gat_comparisons",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "comparisons",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_completion": {
+      "task": "gat_completion",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "completion",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_contextual": {
+      "task": "gat_contextual",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "contextual",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "\u0627\u0648\u062c\u062f \u0627\u0644\u062e\u0637\u0623 \u0627\u0644\u0633\u064a\u0627\u0642\u064a \u0641\u064a \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0627\u0644\u062a\u0627\u0644\u064a\u0629 \u0645\u0646 \u0628\u064a\u0646 \u0627\u0644\u062e\u064a\u0627\u0631\u0627\u062a:",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_geometry": {
+      "task": "gat_geometry",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "geometry",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_reading": {
+      "task": "gat_reading",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "reading",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "gat": 0,
+    "gat_algebra": 0.0,
+    "gat_analogy": 0.0,
+    "gat_arithmetic": 0.0,
+    "gat_association": 0.0,
+    "gat_comparisons": 0.0,
+    "gat_completion": 0.0,
+    "gat_contextual": 0.0,
+    "gat_geometry": 0.0,
+    "gat_reading": 0.0
+  },
+  "n-shot": {
+    "gat_algebra": 0,
+    "gat_analogy": 0,
+    "gat_arithmetic": 0,
+    "gat_association": 0,
+    "gat_comparisons": 0,
+    "gat_completion": 0,
+    "gat_contextual": 0,
+    "gat_geometry": 0,
+    "gat_reading": 0
+  },
+  "higher_is_better": {
+    "gat": {
+      "acc": true
+    },
+    "gat_algebra": {
+      "acc": true
+    },
+    "gat_analogy": {
+      "acc": true
+    },
+    "gat_arithmetic": {
+      "acc": true
+    },
+    "gat_association": {
+      "acc": true
+    },
+    "gat_comparisons": {
+      "acc": true
+    },
+    "gat_completion": {
+      "acc": true
+    },
+    "gat_contextual": {
+      "acc": true
+    },
+    "gat_geometry": {
+      "acc": true
+    },
+    "gat_reading": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "gat_analogy": {
+      "original": 2745,
+      "effective": 2745
+    },
+    "gat_association": {
+      "original": 1045,
+      "effective": 1045
+    },
+    "gat_completion": {
+      "original": 1210,
+      "effective": 1210
+    },
+    "gat_reading": {
+      "original": 2645,
+      "effective": 2645
+    },
+    "gat_algebra": {
+      "original": 2695,
+      "effective": 2695
+    },
+    "gat_arithmetic": {
+      "original": 2717,
+      "effective": 2717
+    },
+    "gat_comparisons": {
+      "original": 1220,
+      "effective": 1220
+    },
+    "gat_contextual": {
+      "original": 1304,
+      "effective": 1304
+    },
+    "gat_geometry": {
+      "original": 365,
+      "effective": 365
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "ef4b2026",
+  "date": 1733932681.9722512,
+  "pretty_env_info": "PyTorch version: 2.1.0a0+29c30b1\nIs debug build: False\nCUDA used to build PyTorch: 12.2\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.22.2\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.1.0a0+29c30b1\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.16.0a0\n[pip3] triton==2.0.0.dev20221202\n[conda] Could not collect",
+  "transformers_version": "4.47.0",
+  "upper_git_hash": "27ba526c4b16ee30604687f8bfd4c19680101dd1",
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 2367.995520754,
+  "end_time": 5482.980996963,
+  "total_evaluation_time_seconds": "3114.9854762089994"
+}

evaluations/ar/AceGPT-v2-32B-Chat/moe_ien_mcq_0_shot.json

@@ -0,0 +1,127 @@
+{
+  "results": {
+    "moe_ien_mcq": {
+      "alias": "moe_ien_mcq",
+      "acc,none": 0.816016016016016,
+      "acc_stderr,none": 0.0038768441643790346,
+      "acc_norm,none": 0.816016016016016,
+      "acc_norm_stderr,none": 0.0038768441643790346
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_mcq": []
+  },
+  "configs": {
+    "moe_ien_mcq": {
+      "task": "moe_ien_mcq",
+      "dataset_path": "lm_eval/tasks/moe_ien_mcq/ien_moe_mcq.py",
+      "dataset_name": "moe_ien_mcq",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.split(\". \", 1)[1] if \". \" in choice else choice\n\n        def format_example(doc, keys):\n            question = doc[\"Question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"Choices\"])]\n                \n            )\n            prompt = f\"\\n\\n\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\"][0:len(doc[\"Choices\"])]\n        out_doc = {\n            \"Query\":  format_example(doc, keys), \n            \"Choices\": keys,\n            \"gold\": int(doc[\"Answer\"])-1, ## \n        }        \n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "Query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{Choices}}",
+      "description": "\u0641\u064a\u0645\u0627\u202f\u064a\u0644\u064a\u202f\u0623\u0633\u0626\u0644\u0629\u202f\u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631\u202f\u0645\u0646\u202f\u0645\u062a\u0639\u062f\u062f\u202f(\u0645\u0639\u202f\u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a)\u202f\u0641\u064a\u202f{{Subject}}",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_mcq": 0.0
+  },
+  "n-shot": {
+    "moe_ien_mcq": 0
+  },
+  "higher_is_better": {
+    "moe_ien_mcq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_mcq": {
+      "original": 9990,
+      "effective": 9990
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738807582.4110897,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {
+    "moe_ien_mcq": "e5422ff2f277b9bfffeb1b5ad185b714804b5a3d276dfff99a29eb88d9a41683"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template_sha": "af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db",
+  "start_time": 1766558.431540363,
+  "end_time": 1766704.504224634,
+  "total_evaluation_time_seconds": "146.07268427102827"
+}

evaluations/ar/AceGPT-v2-32B-Chat/moe_ien_tf_0_shot.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "moe_ien_tf": {
+      "alias": "moe_ien_tf",
+      "acc,none": 0.8035376953460416,
+      "acc_stderr,none": 0.005207228603848848,
+      "acc_norm,none": 0.8035376953460416,
+      "acc_norm_stderr,none": 0.005207228603848848
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_tf": []
+  },
+  "configs": {
+    "moe_ien_tf": {
+      "task": "moe_ien_tf",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/moe_ien_tf/moe_ien_tf.py",
+      "dataset_name": "moe_ien_tf",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        keys=[\"\u0635\u062d\u064a\u062d\u0629\",\n              \"\u062e\u0627\u0637\u0626\u0629\"\n              ]\n        #keys =[\"\u0635\u0648\u0627\u0628\",\n        #         \"\u062e\u0637\u0623\"]\n        target_key = int(doc[\"Answer\"])-1\n\n        out_doc = {\n            \"query\":  \"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" +doc[\"Question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\", \n            \"choices\": keys,\n            \"gold\": target_key,\n        }\n        return out_doc\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{Subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d\u064a\u062d\u0629' \u0623\u0648 '\u062e\u0627\u0637\u0626\u0629' \u062f\u0648\u0646 \u0634\u0631\u062d ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_tf": 2.0
+  },
+  "n-shot": {
+    "moe_ien_tf": 0
+  },
+  "higher_is_better": {
+    "moe_ien_tf": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_tf": {
+      "original": 5823,
+      "effective": 5823
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-32B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 32512545792,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "1c0ca4fb3fa4c292ac3d1f64f330f210c9f184d4",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738809377.2163908,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "151643"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 151643,
+  "max_length": 32768,
+  "task_hashes": {
+    "moe_ien_tf": "116cb28cd11c72b01c3d52d75d3918c312d0a4f569bfdb8b2219398ec576a3f4"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-32B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-32B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template_sha": "af9c0233881b083b52ff773580215222b5440ac3d0beeeca99b76329b048f8db",
+  "start_time": 1768353.06839988,
+  "end_time": 1768502.097875321,
+  "total_evaluation_time_seconds": "149.0294754409697"
+}

evaluations/ar/AceGPT-v2-8B-Chat/acva_5_shot.json

@@ -0,0 +1,123 @@
+{
+  "results": {
+    "acva": {
+      "alias": "acva",
+      "acc,none": 0.7415614236509759,
+      "acc_stderr,none": 0.004691028694524559,
+      "acc_norm,none": 0.7268656716417911,
+      "acc_norm_stderr,none": 0.004774534958083965
+    }
+  },
+  "group_subtasks": {
+    "acva": []
+  },
+  "configs": {
+    "acva": {
+      "task": "acva",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _format_subject(subject):\n        \n        arabic_words = subtasks_ar[subtasks.index(subject)]\n        return arabic_words\n    \n    def _generate_subject(doc):\n        subject = _format_subject(doc[\"id\"].split(\"-\")[0])\n\n        return subject\n        \n    def _process_docs(doc):\n        keys = [\"\u0635\u062d\",\n                 \"\u062e\u0637\u0623\"]\n        subject = _generate_subject(doc)\n        gold = keys.index(doc['answer'])\n        out_doc = {\n            \"id\": doc[\"id\"],\n            \"query\":  \"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" + doc[\"question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\",\n            \"choices\": keys,\n            \"gold\": gold,\n            \"subject\": subject,\n        }\n        \n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d' \u0623\u0648 '\u062e\u0637\u0623' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "acva": 0.0
+  },
+  "n-shot": {
+    "acva": 5
+  },
+  "higher_is_better": {
+    "acva": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "acva": {
+      "original": 8710,
+      "effective": 8710
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 8030261248,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "562d0998c03c02d315e346f81650a43955711901",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "5e10e017",
+  "date": 1736966813.484974,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.0",
+  "upper_git_hash": "2e5cd5395faf76fea1afc96dd0f7161a9d3aa145",
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 2430.929540314,
+  "end_time": 3025.204908665,
+  "total_evaluation_time_seconds": "594.275368351"
+}

evaluations/ar/AceGPT-v2-8B-Chat/ar_ifeval_0_shot.json

@@ -0,0 +1,142 @@
+{
+  "results": {
+    "ar_ifeval": {
+      "alias": "ar_ifeval",
+      "prompt_level_strict_acc,none": 0.10261194029850747,
+      "prompt_level_strict_acc_stderr,none": 0.01311934649092474,
+      "inst_level_strict_acc,none": 0.3924914675767918,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.12126865671641791,
+      "prompt_level_loose_acc_stderr,none": 0.01411319854290401,
+      "inst_level_loose_acc,none": 0.42389078498293514,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "ar_ifeval": []
+  },
+  "configs": {
+    "ar_ifeval": {
+      "task": "ar_ifeval",
+      "dataset_path": "lm_eval/tasks/ar_ifeval/ar_ifeval.py",
+      "dataset_name": "ar_ifeval",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "process_results": "def process_results(doc, results):\n\n    response = results[0]\n    out_strict = process_sample(doc, response, 'strict')\n    out_loose = process_sample(doc, response, 'loose')\n\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0
+      }
+    }
+  },
+  "versions": {
+    "ar_ifeval": 4.0
+  },
+  "n-shot": {
+    "ar_ifeval": 0
+  },
+  "higher_is_better": {
+    "ar_ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ar_ifeval": {
+      "original": 536,
+      "effective": 536
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 8030261248,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "562d0998c03c02d315e346f81650a43955711901",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739784109.8369951,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {
+    "ar_ifeval": "9ce88f26b4b78e684512ecd933af67fe512192f41e27d2bedc62f288943db360"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 62023.729831301,
+  "end_time": 66967.714743853,
+  "total_evaluation_time_seconds": "4943.98491255199"
+}

evaluations/ar/AceGPT-v2-8B-Chat/araMath_v3_5_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "araMath_v3": {
+      "alias": "araMath_v3",
+      "acc,none": 0.41487603305785126,
+      "acc_stderr,none": 0.02004770429343817,
+      "acc_norm,none": 0.41487603305785126,
+      "acc_norm_stderr,none": 0.02004770429343817
+    }
+  },
+  "group_subtasks": {
+    "araMath_v3": []
+  },
+  "configs": {
+    "araMath_v3": {
+      "task": "araMath_v3",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araMath_v3/araMath_v3.py",
+      "dataset_name": "araMath_v3",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def remove_prefix(choice):\n            prefixes = [\"(A)\", \"(B)\", \"(C)\", \"(D)\"]\n            for prefix in prefixes:\n                if choice.startswith(prefix + \" \"):\n                    return choice[len(prefix) + 1:]  \n            return choice \n\n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"options\"])]\n            )\n\n            prompt = f\"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_en.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 \u0628\u064a\u0646 'A\u060c B\u060c C\u060c D' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "araMath_v3": 0.0
+  },
+  "n-shot": {
+    "araMath_v3": 5
+  },
+  "higher_is_better": {
+    "araMath_v3": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araMath_v3": {
+      "original": 605,
+      "effective": 605
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 8030261248,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "562d0998c03c02d315e346f81650a43955711901",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739784015.8084505,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {
+    "araMath_v3": "4eebd1da6e6937fc09bb9f1871adb53192dbce96733f0f8ee76d406c2fc8cad5"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 61929.69246185,
+  "end_time": 61980.464828513,
+  "total_evaluation_time_seconds": "50.772366663004505"
+}

evaluations/ar/AceGPT-v2-8B-Chat/araPro_0_shot.json

@@ -0,0 +1,130 @@
+{
+  "results": {
+    "araPro": {
+      "alias": "araPro",
+      "acc,none": 0.6350729854029195,
+      "acc_stderr,none": 0.006808161111700288,
+      "acc_norm,none": 0.6350729854029195,
+      "acc_norm_stderr,none": 0.006808161111700288
+    }
+  },
+  "group_subtasks": {
+    "araPro": []
+  },
+  "configs": {
+    "araPro": {
+      "task": "araPro",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araPro/araPro.py",
+      "dataset_name": "araPro",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.replace('.', '') if '.' in choice[:2] else choice\n        \n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            \n            choice_num = ['choice1', 'choice2', 'choice3', 'choice4']\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(doc[choice_num[index]])}\\n\" for index, key in enumerate(keys)]\n            )\n\n            prompt = f\"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        #keys = [\"1\", \"2\", \"3\", \"4\"]\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\":  format_example(doc, keys), \n            \"choices\": keys,\n            \"gold\": doc[\"answer\"]-1,\n        }        \n\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question",
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "araPro": 2.0
+  },
+  "n-shot": {
+    "araPro": 0
+  },
+  "higher_is_better": {
+    "araPro": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araPro": {
+      "original": 5001,
+      "effective": 5001
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 8030261248,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "562d0998c03c02d315e346f81650a43955711901",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739782427.4652286,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {
+    "araPro": "655c2f6626c4b10533bba45ff63f9d4501694dea7f65d0bb251390819154f901"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 60341.23142254,
+  "end_time": 60939.383586887,
+  "total_evaluation_time_seconds": "598.1521643470041"
+}

evaluations/ar/AceGPT-v2-8B-Chat/etec_v2_0_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "etec_v2": {
+      "alias": "etec_v2",
+      "acc,none": 0.5680975092739798,
+      "acc_stderr,none": 0.011406002243769559,
+      "acc_norm,none": 0.5680975092739798,
+      "acc_norm_stderr,none": 0.011406002243769559
+    }
+  },
+  "group_subtasks": {
+    "etec_v2": []
+  },
+  "configs": {
+    "etec_v2": {
+      "task": "etec_v2",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/etec_v2/etec.py",
+      "dataset_name": "etec_v2",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n        print(doc[\"label\"])\n        keys_ar = [\"\u0623\", \"\u0628\", \"\u062c\", \"\u062f\"]\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n                \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": int(doc[\"label\"])-1,\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d\n ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "etec_v2": 0.0
+  },
+  "n-shot": {
+    "etec_v2": 0
+  },
+  "higher_is_better": {
+    "etec_v2": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "etec_v2": {
+      "original": 1887,
+      "effective": 1887
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 8030261248,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "562d0998c03c02d315e346f81650a43955711901",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739783073.791851,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {
+    "etec_v2": "d371135bd6f3e91b2eb292576c3b2fae24dc4c0d7cd2a5f6eacf1fe6bc062e76"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 60987.772646854,
+  "end_time": 61072.230445773,
+  "total_evaluation_time_seconds": "84.4577989190002"
+}

evaluations/ar/AceGPT-v2-8B-Chat/exams_ar_5_shot.json

@@ -0,0 +1,119 @@
+{
+  "results": {
+    "exams_ar": {
+      "alias": "exams_ar",
+      "acc,none": 0.5195530726256983,
+      "acc_stderr,none": 0.02158019049784565,
+      "acc_norm,none": 0.5195530726256983,
+      "acc_norm_stderr,none": 0.02158019049784565
+    }
+  },
+  "group_subtasks": {
+    "exams_ar": []
+  },
+  "configs": {
+    "exams_ar": {
+      "task": "exams_ar",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/exams_ar",
+      "dataset_name": "exams_ar",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            \"\"\"\n            <prompt>\n            \u0633\u0624\u0627\u0644:\n            A. <choice1>\n            B. <choice2>\n            C. <choice3>\n            D. <choice4>\n            \u0627\u062c\u0627\u0628\u0629:\n            \"\"\"\n            \n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        def _format_subject(subject):\n            arabic_words = subtasks_ar[subtasks.index(subject)]\n            return arabic_words\n\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n    \n        subject = doc['id'].split(\"-\")[0]\n        description = f\"\ufed2\ufef4\ufee3\ufe8d \ufef2\ufee0\ufef3 \ufe84\ufeb4\ufe8c\ufedf\ufe93 \ufe8d\ufefc\ufea8\ufe98\ufef3\ufe8d\ufead \ufee2\ufee7 \ufee2\ufe98\ufecb\ufea9\ufea9 (\ufee2\ufecb \ufe8d\ufefa\ufe9f\ufe8e\ufe91\ufe8e\ufe97) \ufea1\ufeee\ufedf {_format_subject(subject)} \\n\" #\ufee2\ufee7 \ufed2\ufec0\ufee0\ufedb \ufe8e\ufea8\ufe97\ufead \ufe88\ufe9f\ufe8e\ufe91\ufe93 \ufeed\ufe8e\ufea3\ufea9\ufe93 \ufee2\ufee7 \ufe90\ufef4\ufee7 'A\u060c B\u060c C\u060c D' \ufea9\ufeee\ufee7 \ufeb5\ufeae\ufea3\\n\"\n\n        out_doc = {\n            \"idx\": doc[\"idx\"],\n            \"id\": doc[\"id\"],\n            'dsecription': description,\n            \"query\":  format_example(doc, keys), # \"Question: \" + doc[\"question\"]['stem'] + \"\\nAnswer:\",\n            \"choices\": keys,\n            \"gold\": [\"A\", \"B\", \"C\", \"D\"].index(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "description",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "exams_ar": 0.0
+  },
+  "n-shot": {
+    "exams_ar": 5
+  },
+  "higher_is_better": {
+    "exams_ar": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "exams_ar": {
+      "original": 537,
+      "effective": 537
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.4,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735747770.5687191,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 8055.848670643,
+  "end_time": 8272.25518881,
+  "total_evaluation_time_seconds": "216.40651816700029"
+}

evaluations/ar/AceGPT-v2-8B-Chat/gat_0_shot.json

@@ -0,0 +1,539 @@
+{
+  "results": {
+    "gat": {
+      "acc,none": 0.3615326727706008,
+      "acc_stderr,none": 0.003748588350676633,
+      "alias": "gat"
+    },
+    "gat_algebra": {
+      "alias": " - gat_algebra",
+      "acc,none": 0.30241187384044527,
+      "acc_stderr,none": 0.008849121616191958
+    },
+    "gat_analogy": {
+      "alias": " - gat_analogy",
+      "acc,none": 0.3227686703096539,
+      "acc_stderr,none": 0.008925286248200312
+    },
+    "gat_arithmetic": {
+      "alias": " - gat_arithmetic",
+      "acc,none": 0.3213102686786897,
+      "acc_stderr,none": 0.008960516811645579
+    },
+    "gat_association": {
+      "alias": " - gat_association",
+      "acc,none": 0.39425837320574164,
+      "acc_stderr,none": 0.01512460088966808
+    },
+    "gat_comparisons": {
+      "alias": " - gat_comparisons",
+      "acc,none": 0.28114754098360656,
+      "acc_stderr,none": 0.012876124676937594
+    },
+    "gat_completion": {
+      "alias": " - gat_completion",
+      "acc,none": 0.46115702479338844,
+      "acc_stderr,none": 0.014336474830596175
+    },
+    "gat_contextual": {
+      "alias": " - gat_contextual",
+      "acc,none": 0.2983128834355828,
+      "acc_stderr,none": 0.012674637536976358
+    },
+    "gat_geometry": {
+      "alias": " - gat_geometry",
+      "acc,none": 0.3232876712328767,
+      "acc_stderr,none": 0.024515791774351408
+    },
+    "gat_reading": {
+      "alias": " - gat_reading",
+      "acc,none": 0.5183364839319471,
+      "acc_stderr,none": 0.009717331969425425
+    }
+  },
+  "groups": {
+    "gat": {
+      "acc,none": 0.3615326727706008,
+      "acc_stderr,none": 0.003748588350676633,
+      "alias": "gat"
+    }
+  },
+  "group_subtasks": {
+    "gat": [
+      "gat_analogy",
+      "gat_association",
+      "gat_completion",
+      "gat_reading",
+      "gat_algebra",
+      "gat_arithmetic",
+      "gat_comparisons",
+      "gat_contextual",
+      "gat_geometry"
+    ]
+  },
+  "configs": {
+    "gat_algebra": {
+      "task": "gat_algebra",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_analogy": {
+      "task": "gat_analogy",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "analogy",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_arithmetic": {
+      "task": "gat_arithmetic",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "arithmetic",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_association": {
+      "task": "gat_association",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "association",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_comparisons": {
+      "task": "gat_comparisons",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "comparisons",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_completion": {
+      "task": "gat_completion",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "completion",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_contextual": {
+      "task": "gat_contextual",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "contextual",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "\u0627\u0648\u062c\u062f \u0627\u0644\u062e\u0637\u0623 \u0627\u0644\u0633\u064a\u0627\u0642\u064a \u0641\u064a \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0627\u0644\u062a\u0627\u0644\u064a\u0629 \u0645\u0646 \u0628\u064a\u0646 \u0627\u0644\u062e\u064a\u0627\u0631\u0627\u062a:",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_geometry": {
+      "task": "gat_geometry",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "geometry",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_reading": {
+      "task": "gat_reading",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "reading",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "gat": 0,
+    "gat_algebra": 0.0,
+    "gat_analogy": 0.0,
+    "gat_arithmetic": 0.0,
+    "gat_association": 0.0,
+    "gat_comparisons": 0.0,
+    "gat_completion": 0.0,
+    "gat_contextual": 0.0,
+    "gat_geometry": 0.0,
+    "gat_reading": 0.0
+  },
+  "n-shot": {
+    "gat_algebra": 0,
+    "gat_analogy": 0,
+    "gat_arithmetic": 0,
+    "gat_association": 0,
+    "gat_comparisons": 0,
+    "gat_completion": 0,
+    "gat_contextual": 0,
+    "gat_geometry": 0,
+    "gat_reading": 0
+  },
+  "higher_is_better": {
+    "gat": {
+      "acc": true
+    },
+    "gat_algebra": {
+      "acc": true
+    },
+    "gat_analogy": {
+      "acc": true
+    },
+    "gat_arithmetic": {
+      "acc": true
+    },
+    "gat_association": {
+      "acc": true
+    },
+    "gat_comparisons": {
+      "acc": true
+    },
+    "gat_completion": {
+      "acc": true
+    },
+    "gat_contextual": {
+      "acc": true
+    },
+    "gat_geometry": {
+      "acc": true
+    },
+    "gat_reading": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "gat_analogy": {
+      "original": 2745,
+      "effective": 2745
+    },
+    "gat_association": {
+      "original": 1045,
+      "effective": 1045
+    },
+    "gat_completion": {
+      "original": 1210,
+      "effective": 1210
+    },
+    "gat_reading": {
+      "original": 2645,
+      "effective": 2645
+    },
+    "gat_algebra": {
+      "original": 2695,
+      "effective": 2695
+    },
+    "gat_arithmetic": {
+      "original": 2717,
+      "effective": 2717
+    },
+    "gat_comparisons": {
+      "original": 1220,
+      "effective": 1220
+    },
+    "gat_contextual": {
+      "original": 1304,
+      "effective": 1304
+    },
+    "gat_geometry": {
+      "original": 365,
+      "effective": 365
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.4,download_dir=/tmp",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735749781.6371627,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {},
+  "model_source": "vllm",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 10066.91226392,
+  "end_time": 10586.891967311,
+  "total_evaluation_time_seconds": "519.9797033909999"
+}

evaluations/ar/AceGPT-v2-8B-Chat/moe_ien_mcq_0_shot.json

@@ -0,0 +1,127 @@
+{
+  "results": {
+    "moe_ien_mcq": {
+      "alias": "moe_ien_mcq",
+      "acc,none": 0.7700700700700701,
+      "acc_stderr,none": 0.0042101916833611345,
+      "acc_norm,none": 0.7700700700700701,
+      "acc_norm_stderr,none": 0.0042101916833611345
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_mcq": []
+  },
+  "configs": {
+    "moe_ien_mcq": {
+      "task": "moe_ien_mcq",
+      "dataset_path": "lm_eval/tasks/moe_ien_mcq/ien_moe_mcq.py",
+      "dataset_name": "moe_ien_mcq",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.split(\". \", 1)[1] if \". \" in choice else choice\n\n        def format_example(doc, keys):\n            question = doc[\"Question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"Choices\"])]\n                \n            )\n            prompt = f\"\\n\\n\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\"][0:len(doc[\"Choices\"])]\n        out_doc = {\n            \"Query\":  format_example(doc, keys), \n            \"Choices\": keys,\n            \"gold\": int(doc[\"Answer\"])-1, ## \n        }        \n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "Query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{Choices}}",
+      "description": "\u0641\u064a\u0645\u0627\u202f\u064a\u0644\u064a\u202f\u0623\u0633\u0626\u0644\u0629\u202f\u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631\u202f\u0645\u0646\u202f\u0645\u062a\u0639\u062f\u062f\u202f(\u0645\u0639\u202f\u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a)\u202f\u0641\u064a\u202f{{Subject}}",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_mcq": 0.0
+  },
+  "n-shot": {
+    "moe_ien_mcq": 0
+  },
+  "higher_is_better": {
+    "moe_ien_mcq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_mcq": {
+      "original": 9990,
+      "effective": 9990
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 8030261248,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "562d0998c03c02d315e346f81650a43955711901",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739783202.062394,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {
+    "moe_ien_mcq": "99731f9d1bb76d010da5a439ea1b0bb7695451459d680f708f7222f02ba8e831"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 61116.014324615,
+  "end_time": 61463.567260828,
+  "total_evaluation_time_seconds": "347.5529362130037"
+}

evaluations/ar/AceGPT-v2-8B-Chat/moe_ien_tf_0_shot.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "moe_ien_tf": {
+      "alias": "moe_ien_tf",
+      "acc,none": 0.7590589043448395,
+      "acc_stderr,none": 0.00560476076159517,
+      "acc_norm,none": 0.7590589043448395,
+      "acc_norm_stderr,none": 0.00560476076159517
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_tf": []
+  },
+  "configs": {
+    "moe_ien_tf": {
+      "task": "moe_ien_tf",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/moe_ien_tf/moe_ien_tf.py",
+      "dataset_name": "moe_ien_tf",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        keys=[\"\u0635\u062d\u064a\u062d\u0629\",\n              \"\u062e\u0627\u0637\u0626\u0629\"\n              ]\n        #keys =[\"\u0635\u0648\u0627\u0628\",\n        #         \"\u062e\u0637\u0623\"]\n        target_key = int(doc[\"Answer\"])-1\n\n        out_doc = {\n            \"query\":  \"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" +doc[\"Question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\", \n            \"choices\": keys,\n            \"gold\": target_key,\n        }\n        return out_doc\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{Subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d\u064a\u062d\u0629' \u0623\u0648 '\u062e\u0627\u0637\u0626\u0629' \u062f\u0648\u0646 \u0634\u0631\u062d ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_tf": 2.0
+  },
+  "n-shot": {
+    "moe_ien_tf": 0
+  },
+  "higher_is_better": {
+    "moe_ien_tf": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_tf": {
+      "original": 5823,
+      "effective": 5823
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=FreedomIntelligence/AceGPT-v2-8B-Chat,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 8030261248,
+    "model_dtype": "torch.float16",
+    "model_revision": "main",
+    "model_sha": "562d0998c03c02d315e346f81650a43955711901",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739783594.7150183,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_eos_token": [
+    "<|end_of_text|>",
+    "128001"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128001,
+  "max_length": 8192,
+  "task_hashes": {
+    "moe_ien_tf": "a8315c59ec304a82f04395ff5e7728d6586b1b0b5f569486840b7d29d76a8dd8"
+  },
+  "model_source": "hf",
+  "model_name": "FreedomIntelligence/AceGPT-v2-8B-Chat",
+  "model_name_sanitized": "FreedomIntelligence__AceGPT-v2-8B-Chat",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 61508.598662402,
+  "end_time": 61883.458017876,
+  "total_evaluation_time_seconds": "374.85935547400004"
+}

evaluations/ar/Allam-7b-instruct-preview/acva_5_shot.json

@@ -0,0 +1,119 @@
+{
+  "results": {
+    "acva": {
+      "alias": "acva",
+      "acc,none": 0.7746268656716417,
+      "acc_stderr,none": 0.004477269169728854,
+      "acc_norm,none": 0.7632606199770379,
+      "acc_norm_stderr,none": 0.004554991129754026
+    }
+  },
+  "group_subtasks": {
+    "acva": []
+  },
+  "configs": {
+    "acva": {
+      "task": "acva",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _format_subject(subject):\n        \n        arabic_words = subtasks_ar[subtasks.index(subject)]\n        return arabic_words\n    \n    def _generate_subject(doc):\n        subject = _format_subject(doc[\"id\"].split(\"-\")[0])\n\n        return subject\n        \n    def _process_docs(doc):\n        keys = [\"\u0635\u062d\",\n                 \"\u062e\u0637\u0623\"]\n        subject = _generate_subject(doc)\n        gold = keys.index(doc['answer'])\n        out_doc = {\n            \"id\": doc[\"id\"],\n            \"query\":  \"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" + doc[\"question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\",\n            \"choices\": keys,\n            \"gold\": gold,\n            \"subject\": subject,\n        }\n        \n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d' \u0623\u0648 '\u062e\u0637\u0623' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "acva": 0.0
+  },
+  "n-shot": {
+    "acva": 5
+  },
+  "higher_is_better": {
+    "acva": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "acva": {
+      "original": 8710,
+      "effective": 8710
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735662713.7617116,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "acva": "d007c508f0accdd697f549d7cbe7f960f1470c8f86f1a0969355a6ef33108edb"
+  },
+  "model_source": "vllm",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 3374.021232778,
+  "end_time": 3578.563943596,
+  "total_evaluation_time_seconds": "204.54271081800016"
+}

evaluations/ar/Allam-7b-instruct-preview/ar_ifeval_0_shot.json

@@ -0,0 +1,142 @@
+{
+  "results": {
+    "ar_ifeval": {
+      "alias": "ar_ifeval",
+      "prompt_level_strict_acc,none": 0.31343283582089554,
+      "prompt_level_strict_acc_stderr,none": 0.020055655889994813,
+      "inst_level_strict_acc,none": 0.6764505119453925,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.3656716417910448,
+      "prompt_level_loose_acc_stderr,none": 0.020822161638297296,
+      "inst_level_loose_acc,none": 0.7051194539249147,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "ar_ifeval": []
+  },
+  "configs": {
+    "ar_ifeval": {
+      "task": "ar_ifeval",
+      "dataset_path": "lm_eval/tasks/ar_ifeval/ar_ifeval.py",
+      "dataset_name": "ar_ifeval",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "process_results": "def process_results(doc, results):\n\n    response = results[0]\n    out_strict = process_sample(doc, response, 'strict')\n    out_loose = process_sample(doc, response, 'loose')\n\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0
+      }
+    }
+  },
+  "versions": {
+    "ar_ifeval": 4.0
+  },
+  "n-shot": {
+    "ar_ifeval": 0
+  },
+  "higher_is_better": {
+    "ar_ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ar_ifeval": {
+      "original": 536,
+      "effective": 536
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739618378.981141,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "ar_ifeval": "d0db7903ef270d7dc54efe4e7713be0de9864fc3a36c901c6e5777a6a5f69aa9"
+  },
+  "model_source": "hf",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1393068.333905473,
+  "end_time": 1397143.169266589,
+  "total_evaluation_time_seconds": "4074.8353611161"
+}

evaluations/ar/Allam-7b-instruct-preview/araMath_v3_5_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "araMath_v3": {
+      "alias": "araMath_v3",
+      "acc,none": 0.6677685950413224,
+      "acc_stderr,none": 0.019165266705090528,
+      "acc_norm,none": 0.6677685950413224,
+      "acc_norm_stderr,none": 0.019165266705090528
+    }
+  },
+  "group_subtasks": {
+    "araMath_v3": []
+  },
+  "configs": {
+    "araMath_v3": {
+      "task": "araMath_v3",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araMath_v3/araMath_v3.py",
+      "dataset_name": "araMath_v3",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def remove_prefix(choice):\n            prefixes = [\"(A)\", \"(B)\", \"(C)\", \"(D)\"]\n            for prefix in prefixes:\n                if choice.startswith(prefix + \" \"):\n                    return choice[len(prefix) + 1:]  \n            return choice \n\n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"options\"])]\n            )\n\n            prompt = f\"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_en.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 \u0628\u064a\u0646 'A\u060c B\u060c C\u060c D' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "araMath_v3": 0.0
+  },
+  "n-shot": {
+    "araMath_v3": 5
+  },
+  "higher_is_better": {
+    "araMath_v3": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araMath_v3": {
+      "original": 605,
+      "effective": 605
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739618269.6292942,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "araMath_v3": "e7f60b63c44ee90c76a61f37207fa1f812622b6662200911fcfd7dabe78ada66"
+  },
+  "model_source": "hf",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1392959.193182268,
+  "end_time": 1393012.133225703,
+  "total_evaluation_time_seconds": "52.940043434966356"
+}

evaluations/ar/Allam-7b-instruct-preview/araPro_0_shot.json

@@ -0,0 +1,130 @@
+{
+  "results": {
+    "araPro": {
+      "alias": "araPro",
+      "acc,none": 0.6970605878824235,
+      "acc_stderr,none": 0.006498724870364006,
+      "acc_norm,none": 0.6970605878824235,
+      "acc_norm_stderr,none": 0.006498724870364006
+    }
+  },
+  "group_subtasks": {
+    "araPro": []
+  },
+  "configs": {
+    "araPro": {
+      "task": "araPro",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araPro/araPro.py",
+      "dataset_name": "araPro",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.replace('.', '') if '.' in choice[:2] else choice\n        \n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            \n            choice_num = ['choice1', 'choice2', 'choice3', 'choice4']\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(doc[choice_num[index]])}\\n\" for index, key in enumerate(keys)]\n            )\n\n            prompt = f\"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        #keys = [\"1\", \"2\", \"3\", \"4\"]\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\":  format_example(doc, keys), \n            \"choices\": keys,\n            \"gold\": doc[\"answer\"]-1,\n        }        \n\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question",
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "araPro": 2.0
+  },
+  "n-shot": {
+    "araPro": 0
+  },
+  "higher_is_better": {
+    "araPro": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araPro": {
+      "original": 5001,
+      "effective": 5001
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617164.0204737,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "araPro": "01340c360a1565c46298c4c24dd3fdfe1ea614c6eef6e4d4f021f1da83da2584"
+  },
+  "model_source": "hf",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1391853.516943726,
+  "end_time": 1392050.054185297,
+  "total_evaluation_time_seconds": "196.5372415711172"
+}

evaluations/ar/Allam-7b-instruct-preview/etec_v2_0_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "etec_v2": {
+      "alias": "etec_v2",
+      "acc,none": 0.6666666666666666,
+      "acc_stderr,none": 0.010854826817097195,
+      "acc_norm,none": 0.6666666666666666,
+      "acc_norm_stderr,none": 0.010854826817097195
+    }
+  },
+  "group_subtasks": {
+    "etec_v2": []
+  },
+  "configs": {
+    "etec_v2": {
+      "task": "etec_v2",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/etec_v2/etec.py",
+      "dataset_name": "etec_v2",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n        print(doc[\"label\"])\n        keys_ar = [\"\u0623\", \"\u0628\", \"\u062c\", \"\u062f\"]\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n                \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": int(doc[\"label\"])-1,\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d\n ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "etec_v2": 0.0
+  },
+  "n-shot": {
+    "etec_v2": 0
+  },
+  "higher_is_better": {
+    "etec_v2": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "etec_v2": {
+      "original": 1887,
+      "effective": 1887
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617421.4265695,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "etec_v2": "a0d87bf7eb82815b66ea544cb632aafb803526dee24b399f30fdc751be442b60"
+  },
+  "model_source": "hf",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1392110.980523203,
+  "end_time": 1392198.883363127,
+  "total_evaluation_time_seconds": "87.90283992397599"
+}

evaluations/ar/Allam-7b-instruct-preview/exams_ar_5_shot.json

@@ -0,0 +1,121 @@
+{
+  "results": {
+    "exams_ar": {
+      "alias": "exams_ar",
+      "acc,none": 0.515828677839851,
+      "acc_stderr,none": 0.021585885942816244,
+      "acc_norm,none": 0.515828677839851,
+      "acc_norm_stderr,none": 0.021585885942816244
+    }
+  },
+  "group_subtasks": {
+    "exams_ar": []
+  },
+  "configs": {
+    "exams_ar": {
+      "task": "exams_ar",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/exams_ar",
+      "dataset_name": "exams_ar",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            \"\"\"\n            <prompt>\n            \u0633\u0624\u0627\u0644:\n            A. <choice1>\n            B. <choice2>\n            C. <choice3>\n            D. <choice4>\n            \u0627\u062c\u0627\u0628\u0629:\n            \"\"\"\n            \n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        def _format_subject(subject):\n            arabic_words = subtasks_ar[subtasks.index(subject)]\n            return arabic_words\n\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n    \n        subject = doc['id'].split(\"-\")[0]\n        description = f\"\ufed2\ufef4\ufee3\ufe8d \ufef2\ufee0\ufef3 \ufe84\ufeb4\ufe8c\ufedf\ufe93 \ufe8d\ufefc\ufea8\ufe98\ufef3\ufe8d\ufead \ufee2\ufee7 \ufee2\ufe98\ufecb\ufea9\ufea9 (\ufee2\ufecb \ufe8d\ufefa\ufe9f\ufe8e\ufe91\ufe8e\ufe97) \ufea1\ufeee\ufedf {_format_subject(subject)} \\n\" #\ufee2\ufee7 \ufed2\ufec0\ufee0\ufedb \ufe8e\ufea8\ufe97\ufead \ufe88\ufe9f\ufe8e\ufe91\ufe93 \ufeed\ufe8e\ufea3\ufea9\ufe93 \ufee2\ufee7 \ufe90\ufef4\ufee7 'A\u060c B\u060c C\u060c D' \ufea9\ufeee\ufee7 \ufeb5\ufeae\ufea3\\n\"\n\n        out_doc = {\n            \"idx\": doc[\"idx\"],\n            \"id\": doc[\"id\"],\n            'dsecription': description,\n            \"query\":  format_example(doc, keys), # \"Question: \" + doc[\"question\"]['stem'] + \"\\nAnswer:\",\n            \"choices\": keys,\n            \"gold\": [\"A\", \"B\", \"C\", \"D\"].index(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "description",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "exams_ar": 0.0
+  },
+  "n-shot": {
+    "exams_ar": 5
+  },
+  "higher_is_better": {
+    "exams_ar": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "exams_ar": {
+      "original": 537,
+      "effective": 537
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735662207.0830526,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "exams_ar": "b1561abd56354d570ac16bf64163b0ee8dc6c507234b05f678576b09c26c644a"
+  },
+  "model_source": "vllm",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 2867.397536365,
+  "end_time": 2948.510496752,
+  "total_evaluation_time_seconds": "81.11296038699993"
+}

evaluations/ar/Allam-7b-instruct-preview/gat_0_shot.json

@@ -0,0 +1,549 @@
+{
+  "results": {
+    "gat": {
+      "acc,none": 0.4452527279568544,
+      "acc_stderr,none": 0.0038711388833064567,
+      "alias": "gat"
+    },
+    "gat_algebra": {
+      "alias": " - gat_algebra",
+      "acc,none": 0.40667903525046384,
+      "acc_stderr,none": 0.009463939247454995
+    },
+    "gat_analogy": {
+      "alias": " - gat_analogy",
+      "acc,none": 0.35919854280510016,
+      "acc_stderr,none": 0.009158766245747282
+    },
+    "gat_arithmetic": {
+      "alias": " - gat_arithmetic",
+      "acc,none": 0.40154582259845417,
+      "acc_stderr,none": 0.009406284814832203
+    },
+    "gat_association": {
+      "alias": " - gat_association",
+      "acc,none": 0.5464114832535886,
+      "acc_stderr,none": 0.015407801869520031
+    },
+    "gat_comparisons": {
+      "alias": " - gat_comparisons",
+      "acc,none": 0.34508196721311474,
+      "acc_stderr,none": 0.013616100682624904
+    },
+    "gat_completion": {
+      "alias": " - gat_completion",
+      "acc,none": 0.6057851239669422,
+      "acc_stderr,none": 0.014054411207805699
+    },
+    "gat_contextual": {
+      "alias": " - gat_contextual",
+      "acc,none": 0.3941717791411043,
+      "acc_stderr,none": 0.013537713096332765
+    },
+    "gat_geometry": {
+      "alias": " - gat_geometry",
+      "acc,none": 0.473972602739726,
+      "acc_stderr,none": 0.026171590093068537
+    },
+    "gat_reading": {
+      "alias": " - gat_reading",
+      "acc,none": 0.5727788279773157,
+      "acc_stderr,none": 0.009620311542503682
+    }
+  },
+  "groups": {
+    "gat": {
+      "acc,none": 0.4452527279568544,
+      "acc_stderr,none": 0.0038711388833064567,
+      "alias": "gat"
+    }
+  },
+  "group_subtasks": {
+    "gat": [
+      "gat_analogy",
+      "gat_association",
+      "gat_completion",
+      "gat_reading",
+      "gat_algebra",
+      "gat_arithmetic",
+      "gat_comparisons",
+      "gat_contextual",
+      "gat_geometry"
+    ]
+  },
+  "configs": {
+    "gat_algebra": {
+      "task": "gat_algebra",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_analogy": {
+      "task": "gat_analogy",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "analogy",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_arithmetic": {
+      "task": "gat_arithmetic",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "arithmetic",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_association": {
+      "task": "gat_association",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "association",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_comparisons": {
+      "task": "gat_comparisons",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "comparisons",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_completion": {
+      "task": "gat_completion",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "completion",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_contextual": {
+      "task": "gat_contextual",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "contextual",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "\u0627\u0648\u062c\u062f \u0627\u0644\u062e\u0637\u0623 \u0627\u0644\u0633\u064a\u0627\u0642\u064a \u0641\u064a \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0627\u0644\u062a\u0627\u0644\u064a\u0629 \u0645\u0646 \u0628\u064a\u0646 \u0627\u0644\u062e\u064a\u0627\u0631\u0627\u062a:",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_geometry": {
+      "task": "gat_geometry",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "geometry",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_reading": {
+      "task": "gat_reading",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "reading",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "gat": 0,
+    "gat_algebra": 0.0,
+    "gat_analogy": 0.0,
+    "gat_arithmetic": 0.0,
+    "gat_association": 0.0,
+    "gat_comparisons": 0.0,
+    "gat_completion": 0.0,
+    "gat_contextual": 0.0,
+    "gat_geometry": 0.0,
+    "gat_reading": 0.0
+  },
+  "n-shot": {
+    "gat_algebra": 0,
+    "gat_analogy": 0,
+    "gat_arithmetic": 0,
+    "gat_association": 0,
+    "gat_comparisons": 0,
+    "gat_completion": 0,
+    "gat_contextual": 0,
+    "gat_geometry": 0,
+    "gat_reading": 0
+  },
+  "higher_is_better": {
+    "gat": {
+      "acc": true
+    },
+    "gat_algebra": {
+      "acc": true
+    },
+    "gat_analogy": {
+      "acc": true
+    },
+    "gat_arithmetic": {
+      "acc": true
+    },
+    "gat_association": {
+      "acc": true
+    },
+    "gat_comparisons": {
+      "acc": true
+    },
+    "gat_completion": {
+      "acc": true
+    },
+    "gat_contextual": {
+      "acc": true
+    },
+    "gat_geometry": {
+      "acc": true
+    },
+    "gat_reading": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "gat_analogy": {
+      "original": 2745,
+      "effective": 2745
+    },
+    "gat_association": {
+      "original": 1045,
+      "effective": 1045
+    },
+    "gat_completion": {
+      "original": 1210,
+      "effective": 1210
+    },
+    "gat_reading": {
+      "original": 2645,
+      "effective": 2645
+    },
+    "gat_algebra": {
+      "original": 2695,
+      "effective": 2695
+    },
+    "gat_arithmetic": {
+      "original": 2717,
+      "effective": 2717
+    },
+    "gat_comparisons": {
+      "original": 1220,
+      "effective": 1220
+    },
+    "gat_contextual": {
+      "original": 1304,
+      "effective": 1304
+    },
+    "gat_geometry": {
+      "original": 365,
+      "effective": 365
+    }
+  },
+  "config": {
+    "model": "vllm",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,tensor_parallel_size=1,data_parallel_size=2,gpu_memory_utilization=0.8",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "8e1bd48d",
+  "date": 1735664096.2650902,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100 80GB PCIe\nGPU 1: NVIDIA A100 80GB PCIe\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             48\nOn-line CPU(s) list:                0-47\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V13 64-Core Processor\nCPU family:                         25\nModel:                              1\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          1\nStepping:                           1\nBogoMIPS:                           4890.87\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core invpcid_single vmmcall fsgsbase bmi1 avx2 smep bmi2 erms invpcid rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 xsaves clzero xsaveerptr rdpru arat umip vaes vpclmulqdq rdpid fsrm\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          1.5 MiB (48 instances)\nL1i cache:                          1.5 MiB (48 instances)\nL2 cache:                           24 MiB (48 instances)\nL3 cache:                           192 MiB (6 instances)\nNUMA node(s):                       2\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Not affected\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Vulnerable\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.47.1",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "gat_analogy": "ede28dec097bfebe8a85a19fa27d001696858276df66254bdb70fc63231f1a83",
+    "gat_association": "5d82550d46c4f3cabf370185a8a23cc2eb5b08f1f0c5e210a8a712562a44bd08",
+    "gat_completion": "fc3c19dd7f1896696fec1bffc21182804c9b2f1fb8d8c882428a6bb4bb61e370",
+    "gat_reading": "93053b187a750d2e87f5488f2d0fda944f3da9195bb04d1c4dee9c4b56fa626a",
+    "gat_algebra": "77832c595eaaf156775c3dbb27da0915ef600ebf46a7113ae32a202b0359e8a6",
+    "gat_arithmetic": "6a498f75f5cc0ffd1b30f7a6293ba80d08f2a8876d5558d8e934bf57355ff0cc",
+    "gat_comparisons": "acb80c0ed8dd07e916a471189aef3a546efc289824b2cc50a32c11dc4c97c9c1",
+    "gat_contextual": "de063ed3b94011d74ee24a6532122c9d344fc15e42800db44f0849995a0bc37a",
+    "gat_geometry": "3e482885559a4404ee9e97556edc6e49959770a499f4ae2c58f18ad85b91a363"
+  },
+  "model_source": "vllm",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 4756.376698655,
+  "end_time": 5124.76942052,
+  "total_evaluation_time_seconds": "368.39272186499966"
+}

evaluations/ar/Allam-7b-instruct-preview/moe_ien_mcq_0_shot.json

@@ -0,0 +1,127 @@
+{
+  "results": {
+    "moe_ien_mcq": {
+      "alias": "moe_ien_mcq",
+      "acc,none": 0.9177177177177177,
+      "acc_stderr,none": 0.002749455634736978,
+      "acc_norm,none": 0.9177177177177177,
+      "acc_norm_stderr,none": 0.002749455634736978
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_mcq": []
+  },
+  "configs": {
+    "moe_ien_mcq": {
+      "task": "moe_ien_mcq",
+      "dataset_path": "lm_eval/tasks/moe_ien_mcq/ien_moe_mcq.py",
+      "dataset_name": "moe_ien_mcq",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.split(\". \", 1)[1] if \". \" in choice else choice\n\n        def format_example(doc, keys):\n            question = doc[\"Question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"Choices\"])]\n                \n            )\n            prompt = f\"\\n\\n\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\"][0:len(doc[\"Choices\"])]\n        out_doc = {\n            \"Query\":  format_example(doc, keys), \n            \"Choices\": keys,\n            \"gold\": int(doc[\"Answer\"])-1, ## \n        }        \n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "Query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{Choices}}",
+      "description": "\u0641\u064a\u0645\u0627\u202f\u064a\u0644\u064a\u202f\u0623\u0633\u0626\u0644\u0629\u202f\u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631\u202f\u0645\u0646\u202f\u0645\u062a\u0639\u062f\u062f\u202f(\u0645\u0639\u202f\u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a)\u202f\u0641\u064a\u202f{{Subject}}",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_mcq": 0.0
+  },
+  "n-shot": {
+    "moe_ien_mcq": 0
+  },
+  "higher_is_better": {
+    "moe_ien_mcq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_mcq": {
+      "original": 9990,
+      "effective": 9990
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617571.8184838,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "moe_ien_mcq": "504533b140426f12c89d975ef421328fc89d69af8719c420a1bf897ed4724191"
+  },
+  "model_source": "hf",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1392261.292633723,
+  "end_time": 1392626.942167409,
+  "total_evaluation_time_seconds": "365.64953368599527"
+}

evaluations/ar/Allam-7b-instruct-preview/moe_ien_tf_0_shot.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "moe_ien_tf": {
+      "alias": "moe_ien_tf",
+      "acc,none": 0.8294693456980937,
+      "acc_stderr,none": 0.004929073554117403,
+      "acc_norm,none": 0.8294693456980937,
+      "acc_norm_stderr,none": 0.004929073554117403
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_tf": []
+  },
+  "configs": {
+    "moe_ien_tf": {
+      "task": "moe_ien_tf",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/moe_ien_tf/moe_ien_tf.py",
+      "dataset_name": "moe_ien_tf",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        keys=[\"\u0635\u062d\u064a\u062d\u0629\",\n              \"\u062e\u0627\u0637\u0626\u0629\"\n              ]\n        #keys =[\"\u0635\u0648\u0627\u0628\",\n        #         \"\u062e\u0637\u0623\"]\n        target_key = int(doc[\"Answer\"])-1\n\n        out_doc = {\n            \"query\":  \"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" +doc[\"Question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\", \n            \"choices\": keys,\n            \"gold\": target_key,\n        }\n        return out_doc\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{Subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d\u064a\u062d\u0629' \u0623\u0648 '\u062e\u0627\u0637\u0626\u0629' \u062f\u0648\u0646 \u0634\u0631\u062d ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_tf": 2.0
+  },
+  "n-shot": {
+    "moe_ien_tf": 0
+  },
+  "higher_is_better": {
+    "moe_ien_tf": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_tf": {
+      "original": 5823,
+      "effective": 5823
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=/tmp/7b-alpha-v1.27.2.25,trust_remote_code=True,cache_dir=/tmp,parallelize=False",
+    "model_num_parameters": 7000559616,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617995.3462336,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<unk>",
+    "0"
+  ],
+  "tokenizer_eos_token": [
+    "</s>",
+    "2"
+  ],
+  "tokenizer_bos_token": [
+    "<s>",
+    "1"
+  ],
+  "eot_token_id": 2,
+  "max_length": 4096,
+  "task_hashes": {
+    "moe_ien_tf": "8701a646f6ea8b9bb96c028f817fbeabfb9031580f5054368b43d14d4a5a1270"
+  },
+  "model_source": "hf",
+  "model_name": "/tmp/7b-alpha-v1.27.2.25",
+  "model_name_sanitized": "__tmp__7b-alpha-v1.27.2.25",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + ' [INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
+  "chat_template_sha": "f1dff938141b507da4a409b6bb3431382088a97a963acd246a41f2f344ae831f",
+  "start_time": 1392684.818305694,
+  "end_time": 1392900.218863064,
+  "total_evaluation_time_seconds": "215.40055736992508"
+}

evaluations/ar/Falcon3-7B-Instruct/acva_5_shot.json

@@ -0,0 +1,123 @@
+{
+  "results": {
+    "acva": {
+      "alias": "acva",
+      "acc,none": 0.6045924225028703,
+      "acc_stderr,none": 0.00523925695392083,
+      "acc_norm,none": 0.5897818599311137,
+      "acc_norm_stderr,none": 0.005270708411925859
+    }
+  },
+  "group_subtasks": {
+    "acva": []
+  },
+  "configs": {
+    "acva": {
+      "task": "acva",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _format_subject(subject):\n        \n        arabic_words = subtasks_ar[subtasks.index(subject)]\n        return arabic_words\n    \n    def _generate_subject(doc):\n        subject = _format_subject(doc[\"id\"].split(\"-\")[0])\n\n        return subject\n        \n    def _process_docs(doc):\n        keys = [\"\u0635\u062d\",\n                 \"\u062e\u0637\u0623\"]\n        subject = _generate_subject(doc)\n        gold = keys.index(doc['answer'])\n        out_doc = {\n            \"id\": doc[\"id\"],\n            \"query\":  \"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" + doc[\"question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\",\n            \"choices\": keys,\n            \"gold\": gold,\n            \"subject\": subject,\n        }\n        \n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d' \u0623\u0648 '\u062e\u0637\u0623' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "acva": 0.0
+  },
+  "n-shot": {
+    "acva": 5
+  },
+  "higher_is_better": {
+    "acva": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "acva": {
+      "original": 8710,
+      "effective": 8710
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "5e10e017",
+  "date": 1736889821.9957027,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.0",
+  "upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "acva": "f573ae5740e68711d257f2dc4a23db7c6b1c04895364f1af4b4eb64bfab793a4"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 600072.370318618,
+  "end_time": 600217.222010416,
+  "total_evaluation_time_seconds": "144.85169179795776"
+}

evaluations/ar/Falcon3-7B-Instruct/ar_ifeval_0_shot.json

@@ -0,0 +1,142 @@
+{
+  "results": {
+    "ar_ifeval": {
+      "alias": "ar_ifeval",
+      "prompt_level_strict_acc,none": 0.08582089552238806,
+      "prompt_level_strict_acc_stderr,none": 0.012109752724743699,
+      "inst_level_strict_acc,none": 0.47918088737201364,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.13805970149253732,
+      "prompt_level_loose_acc_stderr,none": 0.014914035308708435,
+      "inst_level_loose_acc,none": 0.5276450511945392,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "ar_ifeval": []
+  },
+  "configs": {
+    "ar_ifeval": {
+      "task": "ar_ifeval",
+      "dataset_path": "lm_eval/tasks/ar_ifeval/ar_ifeval.py",
+      "dataset_name": "ar_ifeval",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "process_results": "def process_results(doc, results):\n\n    response = results[0]\n    out_strict = process_sample(doc, response, 'strict')\n    out_loose = process_sample(doc, response, 'loose')\n\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0
+      }
+    }
+  },
+  "versions": {
+    "ar_ifeval": 4.0
+  },
+  "n-shot": {
+    "ar_ifeval": 0
+  },
+  "higher_is_better": {
+    "ar_ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ar_ifeval": {
+      "original": 536,
+      "effective": 536
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739621196.897086,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "ar_ifeval": "ca837eed1e9f468712643d1fab81b7b48c88a8799239851476bdc889990e6b41"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{%- if tools %}\n{{- '<|system|>\\n' }}\n{%- if messages[0]['role'] == 'system' %}\n{{- messages[0]['content'] }}\n{%- set remaining_messages = messages[1:] %}\n{%- else %}\n{%- set remaining_messages = messages %}\n{%- endif %}\n{{- 'You are a Falcon assistant skilled in function calling. You are helpful, respectful, and concise.\\n\\n# Tools\\n\\nYou have access to the following functions. You MUST use them to answer questions when needed. For each function call, you MUST return a JSON object inside <tool_call></tool_call> tags.\\n\\n<tools>' + tools|tojson(indent=2) + '</tools>\\n\\n# Output Format\\n\\nYour response MUST follow this format when making function calls:\\n<tool_call>\\n[\\n  {\"name\": \"function_name\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": \"value2\"}},\\n  {\"name\": \"another_function\", \"arguments\": {\"arg\": \"value\"}}\\n]\\n</tool_call>\\nIf no function calls are needed, respond normally without the tool_call tags.\\n' }}\n{%- for message in remaining_messages %}\n{%- if message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if message.content %}\n{{- '<|assistant|>\\n' + message['content'] }}\n{%- endif %}\n{%- if message.tool_calls %}\n{{- '\\n<tool_call>\\n' }}\n{{- message.tool_calls|tojson(indent=2) }}\n{{- '\\n</tool_call>' }}\n{%- endif %}\n{{- eos_token + '\\n' }}\n{%- elif message['role'] == 'tool' %}\n{{- '<|assistant|>\\n<tool_response>\\n' + message['content'] + '\\n</tool_response>\\n' }}\n{%- endif %}\n{%- endfor %}\n{{- '<|assistant|>\\n' if add_generation_prompt }}\n{%- else %}\n{%- for message in messages %}\n{%- if message['role'] == 'system' %}\n{{- '<|system|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if not loop.last %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token + '\\n' }}\n{%- else %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token }}\n{%- endif %}\n{%- endif %}\n{%- if loop.last and add_generation_prompt %}\n{{- '<|assistant|>\\n' }}\n{%- endif %}\n{%- endfor %}\n{%- endif %}",
+  "chat_template_sha": "914ccd80356f5822d1a50d97546e37f60c04ed831fe431aa40346574ec266901",
+  "start_time": 1395880.012817552,
+  "end_time": 1401371.318791154,
+  "total_evaluation_time_seconds": "5491.305973601993"
+}

evaluations/ar/Falcon3-7B-Instruct/araMath_v3_5_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "araMath_v3": {
+      "alias": "araMath_v3",
+      "acc,none": 0.5652892561983471,
+      "acc_stderr,none": 0.020170519477736983,
+      "acc_norm,none": 0.5652892561983471,
+      "acc_norm_stderr,none": 0.020170519477736983
+    }
+  },
+  "group_subtasks": {
+    "araMath_v3": []
+  },
+  "configs": {
+    "araMath_v3": {
+      "task": "araMath_v3",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araMath_v3/araMath_v3.py",
+      "dataset_name": "araMath_v3",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def remove_prefix(choice):\n            prefixes = [\"(A)\", \"(B)\", \"(C)\", \"(D)\"]\n            for prefix in prefixes:\n                if choice.startswith(prefix + \" \"):\n                    return choice[len(prefix) + 1:]  \n            return choice \n\n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"options\"])]\n            )\n\n            prompt = f\"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_en.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 \u0628\u064a\u0646 'A\u060c B\u060c C\u060c D' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "araMath_v3": 0.0
+  },
+  "n-shot": {
+    "araMath_v3": 5
+  },
+  "higher_is_better": {
+    "araMath_v3": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araMath_v3": {
+      "original": 605,
+      "effective": 605
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739621084.921236,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "araMath_v3": "b7e29b20c532c7420cc659c6586d56642070560abff0925ed01ad8f200d8e72b"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{%- if tools %}\n{{- '<|system|>\\n' }}\n{%- if messages[0]['role'] == 'system' %}\n{{- messages[0]['content'] }}\n{%- set remaining_messages = messages[1:] %}\n{%- else %}\n{%- set remaining_messages = messages %}\n{%- endif %}\n{{- 'You are a Falcon assistant skilled in function calling. You are helpful, respectful, and concise.\\n\\n# Tools\\n\\nYou have access to the following functions. You MUST use them to answer questions when needed. For each function call, you MUST return a JSON object inside <tool_call></tool_call> tags.\\n\\n<tools>' + tools|tojson(indent=2) + '</tools>\\n\\n# Output Format\\n\\nYour response MUST follow this format when making function calls:\\n<tool_call>\\n[\\n  {\"name\": \"function_name\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": \"value2\"}},\\n  {\"name\": \"another_function\", \"arguments\": {\"arg\": \"value\"}}\\n]\\n</tool_call>\\nIf no function calls are needed, respond normally without the tool_call tags.\\n' }}\n{%- for message in remaining_messages %}\n{%- if message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if message.content %}\n{{- '<|assistant|>\\n' + message['content'] }}\n{%- endif %}\n{%- if message.tool_calls %}\n{{- '\\n<tool_call>\\n' }}\n{{- message.tool_calls|tojson(indent=2) }}\n{{- '\\n</tool_call>' }}\n{%- endif %}\n{{- eos_token + '\\n' }}\n{%- elif message['role'] == 'tool' %}\n{{- '<|assistant|>\\n<tool_response>\\n' + message['content'] + '\\n</tool_response>\\n' }}\n{%- endif %}\n{%- endfor %}\n{{- '<|assistant|>\\n' if add_generation_prompt }}\n{%- else %}\n{%- for message in messages %}\n{%- if message['role'] == 'system' %}\n{{- '<|system|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if not loop.last %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token + '\\n' }}\n{%- else %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token }}\n{%- endif %}\n{%- endif %}\n{%- if loop.last and add_generation_prompt %}\n{{- '<|assistant|>\\n' }}\n{%- endif %}\n{%- endfor %}\n{%- endif %}",
+  "chat_template_sha": "914ccd80356f5822d1a50d97546e37f60c04ed831fe431aa40346574ec266901",
+  "start_time": 1395768.116667791,
+  "end_time": 1395816.745740765,
+  "total_evaluation_time_seconds": "48.629072973970324"
+}

evaluations/ar/Falcon3-7B-Instruct/araPro_0_shot.json

@@ -0,0 +1,130 @@
+{
+  "results": {
+    "araPro": {
+      "alias": "araPro",
+      "acc,none": 0.41471705658868224,
+      "acc_stderr,none": 0.006967450316480296,
+      "acc_norm,none": 0.41471705658868224,
+      "acc_norm_stderr,none": 0.006967450316480296
+    }
+  },
+  "group_subtasks": {
+    "araPro": []
+  },
+  "configs": {
+    "araPro": {
+      "task": "araPro",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araPro/araPro.py",
+      "dataset_name": "araPro",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.replace('.', '') if '.' in choice[:2] else choice\n        \n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            \n            choice_num = ['choice1', 'choice2', 'choice3', 'choice4']\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(doc[choice_num[index]])}\\n\" for index, key in enumerate(keys)]\n            )\n\n            prompt = f\"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        #keys = [\"1\", \"2\", \"3\", \"4\"]\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\":  format_example(doc, keys), \n            \"choices\": keys,\n            \"gold\": doc[\"answer\"]-1,\n        }        \n\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question",
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "araPro": 2.0
+  },
+  "n-shot": {
+    "araPro": 0
+  },
+  "higher_is_better": {
+    "araPro": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araPro": {
+      "original": 5001,
+      "effective": 5001
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739617143.3614087,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "araPro": "063166ad2e52146b6a051c978bf54b1397281e222da633e81fa50357d2409ee9"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{%- if tools %}\n{{- '<|system|>\\n' }}\n{%- if messages[0]['role'] == 'system' %}\n{{- messages[0]['content'] }}\n{%- set remaining_messages = messages[1:] %}\n{%- else %}\n{%- set remaining_messages = messages %}\n{%- endif %}\n{{- 'You are a Falcon assistant skilled in function calling. You are helpful, respectful, and concise.\\n\\n# Tools\\n\\nYou have access to the following functions. You MUST use them to answer questions when needed. For each function call, you MUST return a JSON object inside <tool_call></tool_call> tags.\\n\\n<tools>' + tools|tojson(indent=2) + '</tools>\\n\\n# Output Format\\n\\nYour response MUST follow this format when making function calls:\\n<tool_call>\\n[\\n  {\"name\": \"function_name\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": \"value2\"}},\\n  {\"name\": \"another_function\", \"arguments\": {\"arg\": \"value\"}}\\n]\\n</tool_call>\\nIf no function calls are needed, respond normally without the tool_call tags.\\n' }}\n{%- for message in remaining_messages %}\n{%- if message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if message.content %}\n{{- '<|assistant|>\\n' + message['content'] }}\n{%- endif %}\n{%- if message.tool_calls %}\n{{- '\\n<tool_call>\\n' }}\n{{- message.tool_calls|tojson(indent=2) }}\n{{- '\\n</tool_call>' }}\n{%- endif %}\n{{- eos_token + '\\n' }}\n{%- elif message['role'] == 'tool' %}\n{{- '<|assistant|>\\n<tool_response>\\n' + message['content'] + '\\n</tool_response>\\n' }}\n{%- endif %}\n{%- endfor %}\n{{- '<|assistant|>\\n' if add_generation_prompt }}\n{%- else %}\n{%- for message in messages %}\n{%- if message['role'] == 'system' %}\n{{- '<|system|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if not loop.last %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token + '\\n' }}\n{%- else %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token }}\n{%- endif %}\n{%- endif %}\n{%- if loop.last and add_generation_prompt %}\n{{- '<|assistant|>\\n' }}\n{%- endif %}\n{%- endfor %}\n{%- endif %}",
+  "chat_template_sha": "914ccd80356f5822d1a50d97546e37f60c04ed831fe431aa40346574ec266901",
+  "start_time": 1391826.416201954,
+  "end_time": 1394850.089034202,
+  "total_evaluation_time_seconds": "3023.672832248034"
+}

evaluations/ar/Falcon3-7B-Instruct/etec_v2_0_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "etec_v2": {
+      "alias": "etec_v2",
+      "acc,none": 0.3751987281399046,
+      "acc_stderr,none": 0.01114886834610489,
+      "acc_norm,none": 0.3751987281399046,
+      "acc_norm_stderr,none": 0.01114886834610489
+    }
+  },
+  "group_subtasks": {
+    "etec_v2": []
+  },
+  "configs": {
+    "etec_v2": {
+      "task": "etec_v2",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/etec_v2/etec.py",
+      "dataset_name": "etec_v2",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n        print(doc[\"label\"])\n        keys_ar = [\"\u0623\", \"\u0628\", \"\u062c\", \"\u062f\"]\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n                \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": int(doc[\"label\"])-1,\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d\n ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "etec_v2": 0.0
+  },
+  "n-shot": {
+    "etec_v2": 0
+  },
+  "higher_is_better": {
+    "etec_v2": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "etec_v2": {
+      "original": 1887,
+      "effective": 1887
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739620236.678696,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "etec_v2": "3a8dc6484af6c9538f122c1bbe5c6866dbe14df841fdf04ab7ff2b6437e8aeae"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{%- if tools %}\n{{- '<|system|>\\n' }}\n{%- if messages[0]['role'] == 'system' %}\n{{- messages[0]['content'] }}\n{%- set remaining_messages = messages[1:] %}\n{%- else %}\n{%- set remaining_messages = messages %}\n{%- endif %}\n{{- 'You are a Falcon assistant skilled in function calling. You are helpful, respectful, and concise.\\n\\n# Tools\\n\\nYou have access to the following functions. You MUST use them to answer questions when needed. For each function call, you MUST return a JSON object inside <tool_call></tool_call> tags.\\n\\n<tools>' + tools|tojson(indent=2) + '</tools>\\n\\n# Output Format\\n\\nYour response MUST follow this format when making function calls:\\n<tool_call>\\n[\\n  {\"name\": \"function_name\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": \"value2\"}},\\n  {\"name\": \"another_function\", \"arguments\": {\"arg\": \"value\"}}\\n]\\n</tool_call>\\nIf no function calls are needed, respond normally without the tool_call tags.\\n' }}\n{%- for message in remaining_messages %}\n{%- if message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if message.content %}\n{{- '<|assistant|>\\n' + message['content'] }}\n{%- endif %}\n{%- if message.tool_calls %}\n{{- '\\n<tool_call>\\n' }}\n{{- message.tool_calls|tojson(indent=2) }}\n{{- '\\n</tool_call>' }}\n{%- endif %}\n{{- eos_token + '\\n' }}\n{%- elif message['role'] == 'tool' %}\n{{- '<|assistant|>\\n<tool_response>\\n' + message['content'] + '\\n</tool_response>\\n' }}\n{%- endif %}\n{%- endfor %}\n{{- '<|assistant|>\\n' if add_generation_prompt }}\n{%- else %}\n{%- for message in messages %}\n{%- if message['role'] == 'system' %}\n{{- '<|system|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if not loop.last %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token + '\\n' }}\n{%- else %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token }}\n{%- endif %}\n{%- endif %}\n{%- if loop.last and add_generation_prompt %}\n{{- '<|assistant|>\\n' }}\n{%- endif %}\n{%- endfor %}\n{%- endif %}",
+  "chat_template_sha": "914ccd80356f5822d1a50d97546e37f60c04ed831fe431aa40346574ec266901",
+  "start_time": 1394919.684315533,
+  "end_time": 1394995.42617788,
+  "total_evaluation_time_seconds": "75.7418623471167"
+}

evaluations/ar/Falcon3-7B-Instruct/exams_ar_5_shot.json

@@ -0,0 +1,125 @@
+{
+  "results": {
+    "exams_ar": {
+      "alias": "exams_ar",
+      "acc,none": 0.31843575418994413,
+      "acc_stderr,none": 0.020122499132803468,
+      "acc_norm,none": 0.31843575418994413,
+      "acc_norm_stderr,none": 0.020122499132803468
+    }
+  },
+  "group_subtasks": {
+    "exams_ar": []
+  },
+  "configs": {
+    "exams_ar": {
+      "task": "exams_ar",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/exams_ar",
+      "dataset_name": "exams_ar",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            \"\"\"\n            <prompt>\n            \u0633\u0624\u0627\u0644:\n            A. <choice1>\n            B. <choice2>\n            C. <choice3>\n            D. <choice4>\n            \u0627\u062c\u0627\u0628\u0629:\n            \"\"\"\n            \n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        def _format_subject(subject):\n            arabic_words = subtasks_ar[subtasks.index(subject)]\n            return arabic_words\n\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n    \n        subject = doc['id'].split(\"-\")[0]\n        description = f\"\ufed2\ufef4\ufee3\ufe8d \ufef2\ufee0\ufef3 \ufe84\ufeb4\ufe8c\ufedf\ufe93 \ufe8d\ufefc\ufea8\ufe98\ufef3\ufe8d\ufead \ufee2\ufee7 \ufee2\ufe98\ufecb\ufea9\ufea9 (\ufee2\ufecb \ufe8d\ufefa\ufe9f\ufe8e\ufe91\ufe8e\ufe97) \ufea1\ufeee\ufedf {_format_subject(subject)} \\n\" #\ufee2\ufee7 \ufed2\ufec0\ufee0\ufedb \ufe8e\ufea8\ufe97\ufead \ufe88\ufe9f\ufe8e\ufe91\ufe93 \ufeed\ufe8e\ufea3\ufea9\ufe93 \ufee2\ufee7 \ufe90\ufef4\ufee7 'A\u060c B\u060c C\u060c D' \ufea9\ufeee\ufee7 \ufeb5\ufeae\ufea3\\n\"\n\n        out_doc = {\n            \"idx\": doc[\"idx\"],\n            \"id\": doc[\"id\"],\n            'dsecription': description,\n            \"query\":  format_example(doc, keys), # \"Question: \" + doc[\"question\"]['stem'] + \"\\nAnswer:\",\n            \"choices\": keys,\n            \"gold\": [\"A\", \"B\", \"C\", \"D\"].index(doc[\"label\"]),\n        }\n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "description",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "exams_ar": 0.0
+  },
+  "n-shot": {
+    "exams_ar": 5
+  },
+  "higher_is_better": {
+    "exams_ar": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "exams_ar": {
+      "original": 537,
+      "effective": 537
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "5e10e017",
+  "date": 1736889028.6416683,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.0",
+  "upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "exams_ar": "f52ab3f14b240558420910fdb453ccb45c945cec187c0e60ea51cf6eff08973a"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 599279.04705073,
+  "end_time": 599692.233103212,
+  "total_evaluation_time_seconds": "413.1860524819931"
+}

evaluations/ar/Falcon3-7B-Instruct/gat_0_shot.json

@@ -0,0 +1,553 @@
+{
+  "results": {
+    "gat": {
+      "acc,none": 0.27994481374639407,
+      "acc_stderr,none": 0.003542796359675536,
+      "alias": "gat"
+    },
+    "gat_algebra": {
+      "alias": " - gat_algebra",
+      "acc,none": 0.2571428571428571,
+      "acc_stderr,none": 0.008420562208967575
+    },
+    "gat_analogy": {
+      "alias": " - gat_analogy",
+      "acc,none": 0.24553734061930782,
+      "acc_stderr,none": 0.008216476082874105
+    },
+    "gat_arithmetic": {
+      "alias": " - gat_arithmetic",
+      "acc,none": 0.26573426573426573,
+      "acc_stderr,none": 0.008475894211016492
+    },
+    "gat_association": {
+      "alias": " - gat_association",
+      "acc,none": 0.24019138755980862,
+      "acc_stderr,none": 0.013221495215360054
+    },
+    "gat_comparisons": {
+      "alias": " - gat_comparisons",
+      "acc,none": 0.319672131147541,
+      "acc_stderr,none": 0.013357022766710734
+    },
+    "gat_completion": {
+      "alias": " - gat_completion",
+      "acc,none": 0.27520661157024795,
+      "acc_stderr,none": 0.012844683062506254
+    },
+    "gat_contextual": {
+      "alias": " - gat_contextual",
+      "acc,none": 0.26993865030674846,
+      "acc_stderr,none": 0.01229815625441917
+    },
+    "gat_geometry": {
+      "alias": " - gat_geometry",
+      "acc,none": 0.2876712328767123,
+      "acc_stderr,none": 0.023726723391354485
+    },
+    "gat_reading": {
+      "alias": " - gat_reading",
+      "acc,none": 0.3568998109640832,
+      "acc_stderr,none": 0.009317121354774414
+    }
+  },
+  "groups": {
+    "gat": {
+      "acc,none": 0.27994481374639407,
+      "acc_stderr,none": 0.003542796359675536,
+      "alias": "gat"
+    }
+  },
+  "group_subtasks": {
+    "gat": [
+      "gat_analogy",
+      "gat_association",
+      "gat_completion",
+      "gat_reading",
+      "gat_algebra",
+      "gat_arithmetic",
+      "gat_comparisons",
+      "gat_contextual",
+      "gat_geometry"
+    ]
+  },
+  "configs": {
+    "gat_algebra": {
+      "task": "gat_algebra",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "algebra",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_analogy": {
+      "task": "gat_analogy",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "analogy",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_arithmetic": {
+      "task": "gat_arithmetic",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "arithmetic",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_association": {
+      "task": "gat_association",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "association",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_comparisons": {
+      "task": "gat_comparisons",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "comparisons",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_completion": {
+      "task": "gat_completion",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "completion",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_contextual": {
+      "task": "gat_contextual",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "contextual",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "\u0627\u0648\u062c\u062f \u0627\u0644\u062e\u0637\u0623 \u0627\u0644\u0633\u064a\u0627\u0642\u064a \u0641\u064a \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0627\u0644\u062a\u0627\u0644\u064a\u0629 \u0645\u0646 \u0628\u064a\u0646 \u0627\u0644\u062e\u064a\u0627\u0631\u0627\u062a:",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_geometry": {
+      "task": "gat_geometry",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "geometry",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    },
+    "gat_reading": {
+      "task": "gat_reading",
+      "dataset_path": "lm_eval/tasks/gat/gat_data/gat.py",
+      "dataset_name": "reading",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    # def _process_doc(doc):\n    \n    #     subject = doc['id'].split(\"-\")[0]\n    #     subject_ar = subtasks_ar[subtasks.index(subject)]\n    #     out_doc = {**doc, 'subject_ar': subject_ar}\n    #     print(subject_ar)\n    #     print(out_doc)\n    #     return out_doc\n\n    return dataset\n",
+      "doc_to_text": "{{question}}\n\u0623. {{choices[0]}}\n\u0628. {{choices[1]}}\n\u062c. {{choices[2]}}\n\u062f. {{choices[3]}}\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:",
+      "doc_to_target": "{{label}}",
+      "doc_to_choice": [
+        "\u0623",
+        "\u0628",
+        "\u062c",
+        "\u062f"
+      ],
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "gat": 0,
+    "gat_algebra": 0.0,
+    "gat_analogy": 0.0,
+    "gat_arithmetic": 0.0,
+    "gat_association": 0.0,
+    "gat_comparisons": 0.0,
+    "gat_completion": 0.0,
+    "gat_contextual": 0.0,
+    "gat_geometry": 0.0,
+    "gat_reading": 0.0
+  },
+  "n-shot": {
+    "gat_algebra": 0,
+    "gat_analogy": 0,
+    "gat_arithmetic": 0,
+    "gat_association": 0,
+    "gat_comparisons": 0,
+    "gat_completion": 0,
+    "gat_contextual": 0,
+    "gat_geometry": 0,
+    "gat_reading": 0
+  },
+  "higher_is_better": {
+    "gat": {
+      "acc": true
+    },
+    "gat_algebra": {
+      "acc": true
+    },
+    "gat_analogy": {
+      "acc": true
+    },
+    "gat_arithmetic": {
+      "acc": true
+    },
+    "gat_association": {
+      "acc": true
+    },
+    "gat_comparisons": {
+      "acc": true
+    },
+    "gat_completion": {
+      "acc": true
+    },
+    "gat_contextual": {
+      "acc": true
+    },
+    "gat_geometry": {
+      "acc": true
+    },
+    "gat_reading": {
+      "acc": true
+    }
+  },
+  "n-samples": {
+    "gat_analogy": {
+      "original": 2745,
+      "effective": 2745
+    },
+    "gat_association": {
+      "original": 1045,
+      "effective": 1045
+    },
+    "gat_completion": {
+      "original": 1210,
+      "effective": 1210
+    },
+    "gat_reading": {
+      "original": 2645,
+      "effective": 2645
+    },
+    "gat_algebra": {
+      "original": 2695,
+      "effective": 2695
+    },
+    "gat_arithmetic": {
+      "original": 2717,
+      "effective": 2717
+    },
+    "gat_comparisons": {
+      "original": 1220,
+      "effective": 1220
+    },
+    "gat_contextual": {
+      "original": 1304,
+      "effective": 1304
+    },
+    "gat_geometry": {
+      "original": 365,
+      "effective": 365
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "5e10e017",
+  "date": 1736891004.0192773,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.0",
+  "upper_git_hash": "f64fe2f2a86055aaecced603b56097fd79201711",
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "gat_analogy": "04ac010c48ed039457058b512b7ac0586c7c76a628da7caaf9aeb8f3e99ae5e3",
+    "gat_association": "2cbd868d220125bfcc54ae738592ad902191e4b7f804ce1772ae29e2d3bb3bf6",
+    "gat_completion": "74cf159ef4a3455a6a0e984fed8e9e9a12f0dc21fde95c2058216c5a711a4d31",
+    "gat_reading": "6f21934e536e7dca65361d01e5cafc27f8070c4f0dccf5a88c1fe071194b78a4",
+    "gat_algebra": "20750c926608570eaf87d29981e5ab49b2b097bd52d7f749c44ab4e175d9fdd2",
+    "gat_arithmetic": "c4b0c73c269d9eb3e8482fbda42e69191c28b95e75e1517d5f9142c6ef410204",
+    "gat_comparisons": "88bc22db186a50cab28938ec1fc332366fa0bc886bc98edf810cc9ae938405db",
+    "gat_contextual": "b8e88ff29b62b54eb834dca696304ca0fe1ce55d5cf7d0a9f0204456e3955be6",
+    "gat_geometry": "229545188469d0512a3297737f4ec7afe88d8a30e7e04f87b4982548e83b1e56"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 601254.206185867,
+  "end_time": 601373.470204397,
+  "total_evaluation_time_seconds": "119.26401853002608"
+}

evaluations/ar/Falcon3-7B-Instruct/moe_ien_mcq_0_shot.json

@@ -0,0 +1,127 @@
+{
+  "results": {
+    "moe_ien_mcq": {
+      "alias": "moe_ien_mcq",
+      "acc,none": 0.5265265265265265,
+      "acc_stderr,none": 0.004995706870392996,
+      "acc_norm,none": 0.5265265265265265,
+      "acc_norm_stderr,none": 0.004995706870392996
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_mcq": []
+  },
+  "configs": {
+    "moe_ien_mcq": {
+      "task": "moe_ien_mcq",
+      "dataset_path": "lm_eval/tasks/moe_ien_mcq/ien_moe_mcq.py",
+      "dataset_name": "moe_ien_mcq",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.split(\". \", 1)[1] if \". \" in choice else choice\n\n        def format_example(doc, keys):\n            question = doc[\"Question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"Choices\"])]\n                \n            )\n            prompt = f\"\\n\\n\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\"][0:len(doc[\"Choices\"])]\n        out_doc = {\n            \"Query\":  format_example(doc, keys), \n            \"Choices\": keys,\n            \"gold\": int(doc[\"Answer\"])-1, ## \n        }        \n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "Query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{Choices}}",
+      "description": "\u0641\u064a\u0645\u0627\u202f\u064a\u0644\u064a\u202f\u0623\u0633\u0626\u0644\u0629\u202f\u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631\u202f\u0645\u0646\u202f\u0645\u062a\u0639\u062f\u062f\u202f(\u0645\u0639\u202f\u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a)\u202f\u0641\u064a\u202f{{Subject}}",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_mcq": 0.0
+  },
+  "n-shot": {
+    "moe_ien_mcq": 0
+  },
+  "higher_is_better": {
+    "moe_ien_mcq": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_mcq": {
+      "original": 9990,
+      "effective": 9990
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739620378.768502,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "moe_ien_mcq": "1ae93edb904d572143b5f36dd5dfcc4b901240916d4735ea328083598c912446"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{%- if tools %}\n{{- '<|system|>\\n' }}\n{%- if messages[0]['role'] == 'system' %}\n{{- messages[0]['content'] }}\n{%- set remaining_messages = messages[1:] %}\n{%- else %}\n{%- set remaining_messages = messages %}\n{%- endif %}\n{{- 'You are a Falcon assistant skilled in function calling. You are helpful, respectful, and concise.\\n\\n# Tools\\n\\nYou have access to the following functions. You MUST use them to answer questions when needed. For each function call, you MUST return a JSON object inside <tool_call></tool_call> tags.\\n\\n<tools>' + tools|tojson(indent=2) + '</tools>\\n\\n# Output Format\\n\\nYour response MUST follow this format when making function calls:\\n<tool_call>\\n[\\n  {\"name\": \"function_name\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": \"value2\"}},\\n  {\"name\": \"another_function\", \"arguments\": {\"arg\": \"value\"}}\\n]\\n</tool_call>\\nIf no function calls are needed, respond normally without the tool_call tags.\\n' }}\n{%- for message in remaining_messages %}\n{%- if message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if message.content %}\n{{- '<|assistant|>\\n' + message['content'] }}\n{%- endif %}\n{%- if message.tool_calls %}\n{{- '\\n<tool_call>\\n' }}\n{{- message.tool_calls|tojson(indent=2) }}\n{{- '\\n</tool_call>' }}\n{%- endif %}\n{{- eos_token + '\\n' }}\n{%- elif message['role'] == 'tool' %}\n{{- '<|assistant|>\\n<tool_response>\\n' + message['content'] + '\\n</tool_response>\\n' }}\n{%- endif %}\n{%- endfor %}\n{{- '<|assistant|>\\n' if add_generation_prompt }}\n{%- else %}\n{%- for message in messages %}\n{%- if message['role'] == 'system' %}\n{{- '<|system|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if not loop.last %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token + '\\n' }}\n{%- else %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token }}\n{%- endif %}\n{%- endif %}\n{%- if loop.last and add_generation_prompt %}\n{{- '<|assistant|>\\n' }}\n{%- endif %}\n{%- endfor %}\n{%- endif %}",
+  "chat_template_sha": "914ccd80356f5822d1a50d97546e37f60c04ed831fe431aa40346574ec266901",
+  "start_time": 1395061.894176973,
+  "end_time": 1395336.684131379,
+  "total_evaluation_time_seconds": "274.78995440597646"
+}

evaluations/ar/Falcon3-7B-Instruct/moe_ien_tf_0_shot.json

@@ -0,0 +1,129 @@
+{
+  "results": {
+    "moe_ien_tf": {
+      "alias": "moe_ien_tf",
+      "acc,none": 0.576335222393955,
+      "acc_stderr,none": 0.006476086786980228,
+      "acc_norm,none": 0.576335222393955,
+      "acc_norm_stderr,none": 0.006476086786980228
+    }
+  },
+  "group_subtasks": {
+    "moe_ien_tf": []
+  },
+  "configs": {
+    "moe_ien_tf": {
+      "task": "moe_ien_tf",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/moe_ien_tf/moe_ien_tf.py",
+      "dataset_name": "moe_ien_tf",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        keys=[\"\u0635\u062d\u064a\u062d\u0629\",\n              \"\u062e\u0627\u0637\u0626\u0629\"\n              ]\n        #keys =[\"\u0635\u0648\u0627\u0628\",\n        #         \"\u062e\u0637\u0623\"]\n        target_key = int(doc[\"Answer\"])-1\n\n        out_doc = {\n            \"query\":  \"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" +doc[\"Question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\", \n            \"choices\": keys,\n            \"gold\": target_key,\n        }\n        return out_doc\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{Subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d\u064a\u062d\u0629' \u0623\u0648 '\u062e\u0627\u0637\u0626\u0629' \u062f\u0648\u0646 \u0634\u0631\u062d ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "moe_ien_tf": 2.0
+  },
+  "n-shot": {
+    "moe_ien_tf": 0
+  },
+  "higher_is_better": {
+    "moe_ien_tf": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "moe_ien_tf": {
+      "original": 5823,
+      "effective": 5823
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=tiiuae/Falcon3-7B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 7455550464,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "5563a370c1848366c7a095bde4bbff2cdb419cc6",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "b955b2950",
+  "date": 1739620722.9521024,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.88\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.3",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|pad|>",
+    "2023"
+  ],
+  "tokenizer_eos_token": [
+    "<|endoftext|>",
+    "11"
+  ],
+  "tokenizer_bos_token": [
+    null,
+    "None"
+  ],
+  "eot_token_id": 11,
+  "max_length": 32768,
+  "task_hashes": {
+    "moe_ien_tf": "ed81617ccb178d095c9a81fef15f5ba8b655782b26d36117f53c38b0a84e62e5"
+  },
+  "model_source": "hf",
+  "model_name": "tiiuae/Falcon3-7B-Instruct",
+  "model_name_sanitized": "tiiuae__Falcon3-7B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{%- if tools %}\n{{- '<|system|>\\n' }}\n{%- if messages[0]['role'] == 'system' %}\n{{- messages[0]['content'] }}\n{%- set remaining_messages = messages[1:] %}\n{%- else %}\n{%- set remaining_messages = messages %}\n{%- endif %}\n{{- 'You are a Falcon assistant skilled in function calling. You are helpful, respectful, and concise.\\n\\n# Tools\\n\\nYou have access to the following functions. You MUST use them to answer questions when needed. For each function call, you MUST return a JSON object inside <tool_call></tool_call> tags.\\n\\n<tools>' + tools|tojson(indent=2) + '</tools>\\n\\n# Output Format\\n\\nYour response MUST follow this format when making function calls:\\n<tool_call>\\n[\\n  {\"name\": \"function_name\", \"arguments\": {\"arg1\": \"value1\", \"arg2\": \"value2\"}},\\n  {\"name\": \"another_function\", \"arguments\": {\"arg\": \"value\"}}\\n]\\n</tool_call>\\nIf no function calls are needed, respond normally without the tool_call tags.\\n' }}\n{%- for message in remaining_messages %}\n{%- if message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if message.content %}\n{{- '<|assistant|>\\n' + message['content'] }}\n{%- endif %}\n{%- if message.tool_calls %}\n{{- '\\n<tool_call>\\n' }}\n{{- message.tool_calls|tojson(indent=2) }}\n{{- '\\n</tool_call>' }}\n{%- endif %}\n{{- eos_token + '\\n' }}\n{%- elif message['role'] == 'tool' %}\n{{- '<|assistant|>\\n<tool_response>\\n' + message['content'] + '\\n</tool_response>\\n' }}\n{%- endif %}\n{%- endfor %}\n{{- '<|assistant|>\\n' if add_generation_prompt }}\n{%- else %}\n{%- for message in messages %}\n{%- if message['role'] == 'system' %}\n{{- '<|system|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'user' %}\n{{- '<|user|>\\n' + message['content'] + '\\n' }}\n{%- elif message['role'] == 'assistant' %}\n{%- if not loop.last %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token + '\\n' }}\n{%- else %}\n{{- '<|assistant|>\\n' + message['content'] + eos_token }}\n{%- endif %}\n{%- endif %}\n{%- if loop.last and add_generation_prompt %}\n{{- '<|assistant|>\\n' }}\n{%- endif %}\n{%- endfor %}\n{%- endif %}",
+  "chat_template_sha": "914ccd80356f5822d1a50d97546e37f60c04ed831fe431aa40346574ec266901",
+  "start_time": 1395406.00589162,
+  "end_time": 1395704.54657667,
+  "total_evaluation_time_seconds": "298.54068504995666"
+}

evaluations/ar/Llama-3.3-70B-Instruct/acva_5_shot.json

@@ -0,0 +1,125 @@
+{
+  "results": {
+    "acva": {
+      "alias": "acva",
+      "acc,none": 0.7847301951779564,
+      "acc_stderr,none": 0.004404205705558861,
+      "acc_norm,none": 0.769345579793341,
+      "acc_norm_stderr,none": 0.004513957617295361
+    }
+  },
+  "group_subtasks": {
+    "acva": []
+  },
+  "configs": {
+    "acva": {
+      "task": "acva",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "FreedomIntelligence/ACVA-Arabic-Cultural-Value-Alignment",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _format_subject(subject):\n        \n        arabic_words = subtasks_ar[subtasks.index(subject)]\n        return arabic_words\n    \n    def _generate_subject(doc):\n        subject = _format_subject(doc[\"id\"].split(\"-\")[0])\n\n        return subject\n        \n    def _process_docs(doc):\n        keys = [\"\u0635\u062d\",\n                 \"\u062e\u0637\u0623\"]\n        subject = _generate_subject(doc)\n        gold = keys.index(doc['answer'])\n        out_doc = {\n            \"id\": doc[\"id\"],\n            \"query\":  \"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644:\" + doc[\"question\"]+\"\\n\u0625\u062c\u0627\u0628\u0629:'\",\n            \"choices\": keys,\n            \"gold\": gold,\n            \"subject\": subject,\n        }\n        \n        return out_doc\n\n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0639\u0628\u0627\u0631\u0627\u062a \u0625\u0645\u0627 \u0635\u062d\u064a\u062d\u0629 \u0623\u0648 \u062e\u0627\u0637\u0626\u0629 \u062d\u0648\u0644 {{subject}}\n \u0627\u0644\u0631\u062c\u0627\u0621 \u062a\u0635\u0646\u064a\u0641 \u0627\u0644\u0639\u0628\u0627\u0631\u0629 \u0625\u0644\u0649 '\u0635\u062d' \u0623\u0648 '\u062e\u0637\u0623' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 1.0
+      }
+    }
+  },
+  "versions": {
+    "acva": 1.0
+  },
+  "n-shot": {
+    "acva": 5
+  },
+  "higher_is_better": {
+    "acva": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "acva": {
+      "original": 8710,
+      "effective": 8710
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 70553706496,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "6f6073b423013f6a7d4d9f39144961bfbfbc386b",
+    "batch_size": "auto",
+    "batch_sizes": [
+      64
+    ],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1737861513.0031924,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.1",
+  "upper_git_hash": "086919bd66f4e15fdcd4b792a7b27a698c1ba091",
+  "tokenizer_pad_token": [
+    "<|finetune_right_pad_id|>",
+    "128004"
+  ],
+  "tokenizer_eos_token": [
+    "<|eot_id|>",
+    "128009"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128009,
+  "max_length": 131072,
+  "task_hashes": {},
+  "model_source": "hf",
+  "model_name": "meta-llama/Llama-3.3-70B-Instruct",
+  "model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": null,
+  "chat_template_sha": null,
+  "start_time": 822799.725415956,
+  "end_time": 824041.525682158,
+  "total_evaluation_time_seconds": "1241.8002662019571"
+}

evaluations/ar/Llama-3.3-70B-Instruct/ar_ifeval_0_shot.json

@@ -0,0 +1,142 @@
+{
+  "results": {
+    "ar_ifeval": {
+      "alias": "ar_ifeval",
+      "prompt_level_strict_acc,none": 0.7089552238805971,
+      "prompt_level_strict_acc_stderr,none": 0.019638685568678992,
+      "inst_level_strict_acc,none": 0.8860068259385665,
+      "inst_level_strict_acc_stderr,none": "N/A",
+      "prompt_level_loose_acc,none": 0.7947761194029851,
+      "prompt_level_loose_acc_stderr,none": 0.017460611985170207,
+      "inst_level_loose_acc,none": 0.9208191126279863,
+      "inst_level_loose_acc_stderr,none": "N/A"
+    }
+  },
+  "group_subtasks": {
+    "ar_ifeval": []
+  },
+  "configs": {
+    "ar_ifeval": {
+      "task": "ar_ifeval",
+      "dataset_path": "lm_eval/tasks/ar_ifeval/ar_ifeval.py",
+      "dataset_name": "ar_ifeval",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "test_split": "test",
+      "doc_to_text": "prompt",
+      "doc_to_target": 0,
+      "process_results": "def process_results(doc, results):\n\n    response = results[0]\n    out_strict = process_sample(doc, response, 'strict')\n    out_loose = process_sample(doc, response, 'loose')\n\n\n    return {\n        \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n        \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n        \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n        \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n    }\n",
+      "description": "",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "prompt_level_strict_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_strict_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        },
+        {
+          "metric": "prompt_level_loose_acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "inst_level_loose_acc",
+          "aggregation": "def agg_inst_level_acc(items):\n    flat_items = [item for sublist in items for item in sublist]\n    inst_level_acc = sum(flat_items) / len(flat_items)\n    return inst_level_acc\n",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "generate_until",
+      "generation_kwargs": {
+        "until": [],
+        "do_sample": false,
+        "temperature": 0.0,
+        "max_gen_toks": 1280
+      },
+      "repeats": 1,
+      "should_decontaminate": false,
+      "metadata": {
+        "version": 4.0
+      }
+    }
+  },
+  "versions": {
+    "ar_ifeval": 4.0
+  },
+  "n-shot": {
+    "ar_ifeval": 0
+  },
+  "higher_is_better": {
+    "ar_ifeval": {
+      "prompt_level_strict_acc": true,
+      "inst_level_strict_acc": true,
+      "prompt_level_loose_acc": true,
+      "inst_level_loose_acc": true
+    }
+  },
+  "n-samples": {
+    "ar_ifeval": {
+      "original": 536,
+      "effective": 536
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 70553706496,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "6f6073b423013f6a7d4d9f39144961bfbfbc386b",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738755018.193393,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|finetune_right_pad_id|>",
+    "128004"
+  ],
+  "tokenizer_eos_token": [
+    "<|eot_id|>",
+    "128009"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128009,
+  "max_length": 131072,
+  "task_hashes": {
+    "ar_ifeval": "6bd5bfb26ee4f5909e16d66ee0e564fb2a5826815f16755272465c9e03f98a20"
+  },
+  "model_source": "hf",
+  "model_name": "meta-llama/Llama-3.3-70B-Instruct",
+  "model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65",
+  "start_time": 744977.123888747,
+  "end_time": 758450.608805326,
+  "total_evaluation_time_seconds": "13473.484916579095"
+}

evaluations/ar/Llama-3.3-70B-Instruct/araMath_v3_5_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "araMath_v3": {
+      "alias": "araMath_v3",
+      "acc,none": 0.7090909090909091,
+      "acc_stderr,none": 0.01848039016780232,
+      "acc_norm,none": 0.7090909090909091,
+      "acc_norm_stderr,none": 0.01848039016780232
+    }
+  },
+  "group_subtasks": {
+    "araMath_v3": []
+  },
+  "configs": {
+    "araMath_v3": {
+      "task": "araMath_v3",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araMath_v3/araMath_v3.py",
+      "dataset_name": "araMath_v3",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def remove_prefix(choice):\n            prefixes = [\"(A)\", \"(B)\", \"(C)\", \"(D)\"]\n            for prefix in prefixes:\n                if choice.startswith(prefix + \" \"):\n                    return choice[len(prefix) + 1:]  \n            return choice \n\n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(choice)}\\n\" for key, choice in zip(keys, doc[\"options\"])]\n            )\n\n            prompt = f\"\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": keys_en.index(doc[\"label\"]),\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u0645\u0646 \u0628\u064a\u0646 'A\u060c B\u060c C\u060c D' \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 5,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "araMath_v3": 0.0
+  },
+  "n-shot": {
+    "araMath_v3": 5
+  },
+  "higher_is_better": {
+    "araMath_v3": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araMath_v3": {
+      "original": 605,
+      "effective": 605
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 70553706496,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "6f6073b423013f6a7d4d9f39144961bfbfbc386b",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738750317.5038416,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|finetune_right_pad_id|>",
+    "128004"
+  ],
+  "tokenizer_eos_token": [
+    "<|eot_id|>",
+    "128009"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128009,
+  "max_length": 131072,
+  "task_hashes": {
+    "araMath_v3": "154ea94d6776e7d3980c98343cec49115ef3dc4dab8897fb4668f68494d55c76"
+  },
+  "model_source": "hf",
+  "model_name": "meta-llama/Llama-3.3-70B-Instruct",
+  "model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65",
+  "start_time": 740276.643313964,
+  "end_time": 740434.169818474,
+  "total_evaluation_time_seconds": "157.5265045099659"
+}

evaluations/ar/Llama-3.3-70B-Instruct/araPro_0_shot.json

@@ -0,0 +1,130 @@
+{
+  "results": {
+    "araPro": {
+      "alias": "araPro",
+      "acc,none": 0.7048590281943611,
+      "acc_stderr,none": 0.006450314388729491,
+      "acc_norm,none": 0.7048590281943611,
+      "acc_norm_stderr,none": 0.006450314388729491
+    }
+  },
+  "group_subtasks": {
+    "araPro": []
+  },
+  "configs": {
+    "araPro": {
+      "task": "araPro",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/araPro/araPro.py",
+      "dataset_name": "araPro",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "fewshot_split": "validation",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):        \n        def remove_prefix(choice):\n            return choice.replace('.', '') if '.' in choice[:2] else choice\n        \n        def format_example(doc, keys):\n            question = doc[\"question\"].strip()\n            \n            choice_num = ['choice1', 'choice2', 'choice3', 'choice4']\n            choices = \"\".join(\n                [f\"{key}. {remove_prefix(doc[choice_num[index]])}\\n\" for index, key in enumerate(keys)]\n            )\n\n            prompt = f\"\\n\\n\\n\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices} \\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n\n        #keys = [\"1\", \"2\", \"3\", \"4\"]\n        keys = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n            \"query\":  format_example(doc, keys), \n            \"choices\": keys,\n            \"gold\": doc[\"answer\"]-1,\n        }        \n\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "{{choices}}",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "fewshot_config": {
+        "sampler": "balanced_cat"
+      },
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "Question",
+      "metadata": {
+        "version": 2.0
+      }
+    }
+  },
+  "versions": {
+    "araPro": 2.0
+  },
+  "n-shot": {
+    "araPro": 0
+  },
+  "higher_is_better": {
+    "araPro": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "araPro": {
+      "original": 5001,
+      "effective": 5001
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 70553706496,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "6f6073b423013f6a7d4d9f39144961bfbfbc386b",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738742514.712935,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|finetune_right_pad_id|>",
+    "128004"
+  ],
+  "tokenizer_eos_token": [
+    "<|eot_id|>",
+    "128009"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128009,
+  "max_length": 131072,
+  "task_hashes": {
+    "araPro": "ab4849e5668de72a27844a2a354787cbce92af5027f46a32300417b41913c5db"
+  },
+  "model_source": "hf",
+  "model_name": "meta-llama/Llama-3.3-70B-Instruct",
+  "model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65",
+  "start_time": 732473.787962617,
+  "end_time": 736407.61692168,
+  "total_evaluation_time_seconds": "3933.8289590630447"
+}

evaluations/ar/Llama-3.3-70B-Instruct/etec_v2_0_shot.json

@@ -0,0 +1,126 @@
+{
+  "results": {
+    "etec_v2": {
+      "alias": "etec_v2",
+      "acc,none": 0.6883942766295708,
+      "acc_stderr,none": 0.010664745454850943,
+      "acc_norm,none": 0.6883942766295708,
+      "acc_norm_stderr,none": 0.010664745454850943
+    }
+  },
+  "group_subtasks": {
+    "etec_v2": []
+  },
+  "configs": {
+    "etec_v2": {
+      "task": "etec_v2",
+      "tag": [
+        "multiple_choice"
+      ],
+      "dataset_path": "lm_eval/tasks/etec_v2/etec.py",
+      "dataset_name": "etec_v2",
+      "dataset_kwargs": {
+        "trust_remote_code": true
+      },
+      "validation_split": "validation",
+      "test_split": "test",
+      "process_docs": "def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\n    def _process_docs(doc):\n        def format_example(doc, keys):\n            question =  doc[\"question\"].strip()\n            \n            choices = \"\".join(\n                [f\"{key}. {choice}\\n\" for key, choice in zip(keys,  doc[\"choices\"])]\n            )\n            prompt = f\"\u0627\u0644\u0633\u0624\u0627\u0644: {question}\\n{choices}\\n\u0627\u0644\u0627\u062c\u0627\u0628\u0629:\"\n            return prompt\n        print(doc[\"label\"])\n        keys_ar = [\"\u0623\", \"\u0628\", \"\u062c\", \"\u062f\"]\n        keys_en = [\"A\", \"B\", \"C\", \"D\"]\n        out_doc = {\n                \"query\": format_example(doc, keys_en),\n            \"choices\": keys_en,\n            \"gold\": int(doc[\"label\"])-1,\n        }\n        return out_doc\n    \n    return dataset.map(_process_docs)\n",
+      "doc_to_text": "query",
+      "doc_to_target": "gold",
+      "doc_to_choice": "choices",
+      "description": "\u0641\u064a\u0645\u0627 \u064a\u0644\u064a \u0623\u0633\u0626\u0644\u0629 \u0627\u0644\u0627\u062e\u062a\u064a\u0627\u0631 \u0645\u0646 \u0645\u062a\u0639\u062f\u062f (\u0645\u0639 \u0627\u0644\u0625\u062c\u0627\u0628\u0627\u062a) \u0645\u0646 \u0641\u0636\u0644\u0643 \u0627\u062e\u062a\u0631 \u0625\u062c\u0627\u0628\u0629 \u0648\u0627\u062d\u062f\u0629 \u062f\u0648\u0646 \u0634\u0631\u062d\n ",
+      "target_delimiter": " ",
+      "fewshot_delimiter": "\n\n",
+      "num_fewshot": 0,
+      "metric_list": [
+        {
+          "metric": "acc",
+          "aggregation": "mean",
+          "higher_is_better": true
+        },
+        {
+          "metric": "acc_norm",
+          "aggregation": "mean",
+          "higher_is_better": true
+        }
+      ],
+      "output_type": "multiple_choice",
+      "repeats": 1,
+      "should_decontaminate": true,
+      "doc_to_decontamination_query": "query",
+      "metadata": {
+        "version": 0.0
+      }
+    }
+  },
+  "versions": {
+    "etec_v2": 0.0
+  },
+  "n-shot": {
+    "etec_v2": 0
+  },
+  "higher_is_better": {
+    "etec_v2": {
+      "acc": true,
+      "acc_norm": true
+    }
+  },
+  "n-samples": {
+    "etec_v2": {
+      "original": 1887,
+      "effective": 1887
+    }
+  },
+  "config": {
+    "model": "hf",
+    "model_args": "pretrained=meta-llama/Llama-3.3-70B-Instruct,trust_remote_code=True,cache_dir=/tmp,parallelize=True",
+    "model_num_parameters": 70553706496,
+    "model_dtype": "torch.bfloat16",
+    "model_revision": "main",
+    "model_sha": "6f6073b423013f6a7d4d9f39144961bfbfbc386b",
+    "batch_size": 1,
+    "batch_sizes": [],
+    "device": null,
+    "use_cache": null,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "gen_kwargs": null,
+    "random_seed": 0,
+    "numpy_seed": 1234,
+    "torch_seed": 1234,
+    "fewshot_seed": 1234
+  },
+  "git_hash": "788a3672",
+  "date": 1738746708.9926562,
+  "pretty_env_info": "PyTorch version: 2.4.0+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Ubuntu 22.04.3 LTS (x86_64)\nGCC version: (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\nClang version: Could not collect\nCMake version: version 3.27.1\nLibc version: glibc-2.35\n\nPython version: 3.10.12 (main, Jun 11 2023, 05:26:28) [GCC 11.4.0] (64-bit runtime)\nPython platform: Linux-5.15.0-1064-azure-x86_64-with-glibc2.35\nIs CUDA available: True\nCUDA runtime version: 12.2.128\nCUDA_MODULE_LOADING set to: LAZY\nGPU models and configuration: \nGPU 0: NVIDIA A100-SXM4-80GB\nGPU 1: NVIDIA A100-SXM4-80GB\nGPU 2: NVIDIA A100-SXM4-80GB\nGPU 3: NVIDIA A100-SXM4-80GB\nGPU 4: NVIDIA A100-SXM4-80GB\nGPU 5: NVIDIA A100-SXM4-80GB\nGPU 6: NVIDIA A100-SXM4-80GB\nGPU 7: NVIDIA A100-SXM4-80GB\n\nNvidia driver version: 535.161.08\ncuDNN version: Probably one of the following:\n/usr/lib/x86_64-linux-gnu/libcudnn.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_adv_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_cnn_train.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_infer.so.8.9.4\n/usr/lib/x86_64-linux-gnu/libcudnn_ops_train.so.8.9.4\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture:                       x86_64\nCPU op-mode(s):                     32-bit, 64-bit\nAddress sizes:                      48 bits physical, 48 bits virtual\nByte Order:                         Little Endian\nCPU(s):                             96\nOn-line CPU(s) list:                0-95\nVendor ID:                          AuthenticAMD\nModel name:                         AMD EPYC 7V12 64-Core Processor\nCPU family:                         23\nModel:                              49\nThread(s) per core:                 1\nCore(s) per socket:                 48\nSocket(s):                          2\nStepping:                           0\nBogoMIPS:                           4890.89\nFlags:                              fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl tsc_reliable nonstop_tsc cpuid extd_apicid aperfmperf pni pclmulqdq ssse3 fma cx16 sse4_1 sse4_2 movbe popcnt aes xsave avx f16c rdrand hypervisor lahf_lm cmp_legacy cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw topoext perfctr_core ssbd vmmcall fsgsbase bmi1 avx2 smep bmi2 rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 clzero xsaveerptr rdpru arat umip rdpid\nHypervisor vendor:                  Microsoft\nVirtualization type:                full\nL1d cache:                          3 MiB (96 instances)\nL1i cache:                          3 MiB (96 instances)\nL2 cache:                           48 MiB (96 instances)\nL3 cache:                           384 MiB (24 instances)\nNUMA node(s):                       4\nNUMA node0 CPU(s):                  0-23\nNUMA node1 CPU(s):                  24-47\nNUMA node2 CPU(s):                  48-71\nNUMA node3 CPU(s):                  72-95\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit:        Not affected\nVulnerability L1tf:                 Not affected\nVulnerability Mds:                  Not affected\nVulnerability Meltdown:             Not affected\nVulnerability Mmio stale data:      Not affected\nVulnerability Retbleed:             Mitigation; untrained return thunk; SMT disabled\nVulnerability Spec rstack overflow: Mitigation; safe RET, no microcode\nVulnerability Spec store bypass:    Mitigation; Speculative Store Bypass disabled via prctl and seccomp\nVulnerability Spectre v1:           Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2:           Mitigation; Retpolines; STIBP disabled; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds:                Not affected\nVulnerability Tsx async abort:      Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] onnx==1.14.0\n[pip3] pytorch-lightning==2.0.7\n[pip3] pytorch-quantization==2.1.2\n[pip3] torch==2.4.0\n[pip3] torch-tensorrt==2.0.0.dev0\n[pip3] torchaudio==2.1.0\n[pip3] torchdata==0.7.0a0\n[pip3] torchmetrics==1.2.0\n[pip3] torchvision==0.19.0\n[pip3] triton==3.0.0\n[conda] Could not collect",
+  "transformers_version": "4.48.2",
+  "upper_git_hash": null,
+  "tokenizer_pad_token": [
+    "<|finetune_right_pad_id|>",
+    "128004"
+  ],
+  "tokenizer_eos_token": [
+    "<|eot_id|>",
+    "128009"
+  ],
+  "tokenizer_bos_token": [
+    "<|begin_of_text|>",
+    "128000"
+  ],
+  "eot_token_id": 128009,
+  "max_length": 131072,
+  "task_hashes": {
+    "etec_v2": "f9810ea40ab4721486631d02578e3b62811871d66f80ee350dc574ca63d72e12"
+  },
+  "model_source": "hf",
+  "model_name": "meta-llama/Llama-3.3-70B-Instruct",
+  "model_name_sanitized": "meta-llama__Llama-3.3-70B-Instruct",
+  "system_instruction": null,
+  "system_instruction_sha": null,
+  "fewshot_as_multiturn": false,
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "chat_template_sha": "e10ca381b1ccc5cf9db52e371f3b6651576caee0a630b452e2816b2d404d4b65",
+  "start_time": 736668.210182346,
+  "end_time": 736927.122919428,
+  "total_evaluation_time_seconds": "258.9127370819915"
+}