Hugging Face's logo

Spaces:
Krisseck
/
IFEval-Leaderboard
Running

App Files Community
1
Krisseck commited on
Commit
fa2b48a
•
1 Parent(s): 040029a

Add results

Browse files
Files changed (5) hide show
  1. results/CapybaraHermes-2.5-Mistral-7B-GGUF.json +117 -0
  2. results/Meta-Llama-3-8B-Instruct-GGUF.json +117 -0
  3. results/Meta-Llama-3-8B-Instruct-abliterated-v3-GGUF.json +117 -0
  4. results/Mistral-7B-Instruct-v0.3-GGUF.json +117 -0
  5. results/Phi-3-medium-4k-instruct-GGUF.json +117 -0
results/CapybaraHermes-2.5-Mistral-7B-GGUF.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "ifeval": {
4
+ "prompt_level_strict_acc,none": 0.47504621072088726,
5
+ "prompt_level_strict_acc_stderr,none": 0.021489761058967286,
6
+ "inst_level_strict_acc,none": 0.5995203836930456,
7
+ "inst_level_strict_acc_stderr,none": "N/A",
8
+ "prompt_level_loose_acc,none": 0.49353049907578556,
9
+ "prompt_level_loose_acc_stderr,none": 0.021514772941787025,
10
+ "inst_level_loose_acc,none": 0.6211031175059952,
11
+ "inst_level_loose_acc_stderr,none": "N/A",
12
+ "alias": "ifeval"
13
+ }
14
+ },
15
+ "group_subtasks": {
16
+ "ifeval": []
17
+ },
18
+ "configs": {
19
+ "ifeval": {
20
+ "task": "ifeval",
21
+ "dataset_path": "wis-k/instruction-following-eval",
22
+ "test_split": "train",
23
+ "doc_to_text": "prompt",
24
+ "doc_to_target": 0,
25
+ "process_results": "def process_results(doc, results):\n eval_logger.warning(\n \"This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models\"\n )\n\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
26
+ "description": "",
27
+ "target_delimiter": " ",
28
+ "fewshot_delimiter": "\n\n",
29
+ "num_fewshot": 0,
30
+ "metric_list": [
31
+ {
32
+ "metric": "prompt_level_strict_acc",
33
+ "aggregation": "mean",
34
+ "higher_is_better": true
35
+ },
36
+ {
37
+ "metric": "inst_level_strict_acc",
38
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
39
+ "higher_is_better": true
40
+ },
41
+ {
42
+ "metric": "prompt_level_loose_acc",
43
+ "aggregation": "mean",
44
+ "higher_is_better": true
45
+ },
46
+ {
47
+ "metric": "inst_level_loose_acc",
48
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
49
+ "higher_is_better": true
50
+ }
51
+ ],
52
+ "output_type": "generate_until",
53
+ "generation_kwargs": {
54
+ "until": [],
55
+ "do_sample": false,
56
+ "temperature": 0.0,
57
+ "max_gen_toks": 1280
58
+ },
59
+ "repeats": 1,
60
+ "should_decontaminate": false,
61
+ "metadata": {
62
+ "version": 2.0
63
+ }
64
+ }
65
+ },
66
+ "versions": {
67
+ "ifeval": 2.0
68
+ },
69
+ "n-shot": {
70
+ "ifeval": 0
71
+ },
72
+ "higher_is_better": {
73
+ "ifeval": {
74
+ "prompt_level_strict_acc": true,
75
+ "inst_level_strict_acc": true,
76
+ "prompt_level_loose_acc": true,
77
+ "inst_level_loose_acc": true
78
+ }
79
+ },
80
+ "n-samples": {
81
+ "ifeval": {
82
+ "original": 541,
83
+ "effective": 541
84
+ }
85
+ },
86
+ "config": {
87
+ "model": "local-chat-completions",
88
+ "model_args": "base_url=http://localhost:7860/v1",
89
+ "batch_size": 1,
90
+ "batch_sizes": [],
91
+ "device": null,
92
+ "use_cache": null,
93
+ "limit": null,
94
+ "bootstrap_iters": 100000,
95
+ "gen_kwargs": null,
96
+ "random_seed": 0,
97
+ "numpy_seed": 1234,
98
+ "torch_seed": 1234,
99
+ "fewshot_seed": 1234
100
+ },
101
+ "git_hash": "e39df01c",
102
+ "date": 1717661416.8740277,
103
+ "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen 7 3700X 8-Core Processor\nCPU family: 23\nModel: 113\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 0\nFrequency boost: enabled\nCPU(s) scaling MHz: 75%\nCPU max MHz: 4426.1709\nCPU min MHz: 2200.0000\nBogoMIPS: 7189.76\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization: AMD-V\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 4 MiB (8 instances)\nL3 cache: 32 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; Safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[conda] Could not collect",
104
+ "transformers_version": "4.41.2",
105
+ "upper_git_hash": null,
106
+ "task_hashes": {},
107
+ "model_source": "local-chat-completions",
108
+ "model_name": "",
109
+ "model_name_sanitized": "",
110
+ "system_instruction": null,
111
+ "system_instruction_sha": null,
112
+ "chat_template": null,
113
+ "chat_template_sha": null,
114
+ "start_time": 604672.281265057,
115
+ "end_time": 607554.9350951,
116
+ "total_evaluation_time_seconds": "2882.653830042924"
117
+ }
results/Meta-Llama-3-8B-Instruct-GGUF.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "ifeval": {
4
+ "prompt_level_strict_acc,none": 0.6765249537892791,
5
+ "prompt_level_strict_acc_stderr,none": 0.020131003392118964,
6
+ "inst_level_strict_acc,none": 0.7613908872901679,
7
+ "inst_level_strict_acc_stderr,none": "N/A",
8
+ "prompt_level_loose_acc,none": 0.7171903881700554,
9
+ "prompt_level_loose_acc_stderr,none": 0.019380609595892707,
10
+ "inst_level_loose_acc,none": 0.7913669064748201,
11
+ "inst_level_loose_acc_stderr,none": "N/A",
12
+ "alias": "ifeval"
13
+ }
14
+ },
15
+ "group_subtasks": {
16
+ "ifeval": []
17
+ },
18
+ "configs": {
19
+ "ifeval": {
20
+ "task": "ifeval",
21
+ "dataset_path": "wis-k/instruction-following-eval",
22
+ "test_split": "train",
23
+ "doc_to_text": "prompt",
24
+ "doc_to_target": 0,
25
+ "process_results": "def process_results(doc, results):\n eval_logger.warning(\n \"This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models\"\n )\n\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
26
+ "description": "",
27
+ "target_delimiter": " ",
28
+ "fewshot_delimiter": "\n\n",
29
+ "num_fewshot": 0,
30
+ "metric_list": [
31
+ {
32
+ "metric": "prompt_level_strict_acc",
33
+ "aggregation": "mean",
34
+ "higher_is_better": true
35
+ },
36
+ {
37
+ "metric": "inst_level_strict_acc",
38
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
39
+ "higher_is_better": true
40
+ },
41
+ {
42
+ "metric": "prompt_level_loose_acc",
43
+ "aggregation": "mean",
44
+ "higher_is_better": true
45
+ },
46
+ {
47
+ "metric": "inst_level_loose_acc",
48
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
49
+ "higher_is_better": true
50
+ }
51
+ ],
52
+ "output_type": "generate_until",
53
+ "generation_kwargs": {
54
+ "until": [],
55
+ "do_sample": false,
56
+ "temperature": 0.0,
57
+ "max_gen_toks": 1280
58
+ },
59
+ "repeats": 1,
60
+ "should_decontaminate": false,
61
+ "metadata": {
62
+ "version": 2.0
63
+ }
64
+ }
65
+ },
66
+ "versions": {
67
+ "ifeval": 2.0
68
+ },
69
+ "n-shot": {
70
+ "ifeval": 0
71
+ },
72
+ "higher_is_better": {
73
+ "ifeval": {
74
+ "prompt_level_strict_acc": true,
75
+ "inst_level_strict_acc": true,
76
+ "prompt_level_loose_acc": true,
77
+ "inst_level_loose_acc": true
78
+ }
79
+ },
80
+ "n-samples": {
81
+ "ifeval": {
82
+ "original": 541,
83
+ "effective": 541
84
+ }
85
+ },
86
+ "config": {
87
+ "model": "local-chat-completions",
88
+ "model_args": "base_url=http://localhost:7860/v1",
89
+ "batch_size": 1,
90
+ "batch_sizes": [],
91
+ "device": null,
92
+ "use_cache": null,
93
+ "limit": null,
94
+ "bootstrap_iters": 100000,
95
+ "gen_kwargs": null,
96
+ "random_seed": 0,
97
+ "numpy_seed": 1234,
98
+ "torch_seed": 1234,
99
+ "fewshot_seed": 1234
100
+ },
101
+ "git_hash": "e39df01c",
102
+ "date": 1717646650.755885,
103
+ "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen 7 3700X 8-Core Processor\nCPU family: 23\nModel: 113\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 0\nFrequency boost: enabled\nCPU(s) scaling MHz: 75%\nCPU max MHz: 4426.1709\nCPU min MHz: 2200.0000\nBogoMIPS: 7189.76\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization: AMD-V\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 4 MiB (8 instances)\nL3 cache: 32 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; Safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[conda] Could not collect",
104
+ "transformers_version": "4.41.2",
105
+ "upper_git_hash": null,
106
+ "task_hashes": {},
107
+ "model_source": "local-chat-completions",
108
+ "model_name": "",
109
+ "model_name_sanitized": "",
110
+ "system_instruction": null,
111
+ "system_instruction_sha": null,
112
+ "chat_template": null,
113
+ "chat_template_sha": null,
114
+ "start_time": 589906.269666123,
115
+ "end_time": 592435.624952695,
116
+ "total_evaluation_time_seconds": "2529.355286572012"
117
+ }
results/Meta-Llama-3-8B-Instruct-abliterated-v3-GGUF.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "ifeval": {
4
+ "prompt_level_strict_acc,none": 0.7005545286506469,
5
+ "prompt_level_strict_acc_stderr,none": 0.019709834029672937,
6
+ "inst_level_strict_acc,none": 0.7817745803357314,
7
+ "inst_level_strict_acc_stderr,none": "N/A",
8
+ "prompt_level_loose_acc,none": 0.7393715341959335,
9
+ "prompt_level_loose_acc_stderr,none": 0.018890584986760186,
10
+ "inst_level_loose_acc,none": 0.8141486810551559,
11
+ "inst_level_loose_acc_stderr,none": "N/A",
12
+ "alias": "ifeval"
13
+ }
14
+ },
15
+ "group_subtasks": {
16
+ "ifeval": []
17
+ },
18
+ "configs": {
19
+ "ifeval": {
20
+ "task": "ifeval",
21
+ "dataset_path": "wis-k/instruction-following-eval",
22
+ "test_split": "train",
23
+ "doc_to_text": "prompt",
24
+ "doc_to_target": 0,
25
+ "process_results": "def process_results(doc, results):\n eval_logger.warning(\n \"This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models\"\n )\n\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
26
+ "description": "",
27
+ "target_delimiter": " ",
28
+ "fewshot_delimiter": "\n\n",
29
+ "num_fewshot": 0,
30
+ "metric_list": [
31
+ {
32
+ "metric": "prompt_level_strict_acc",
33
+ "aggregation": "mean",
34
+ "higher_is_better": true
35
+ },
36
+ {
37
+ "metric": "inst_level_strict_acc",
38
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
39
+ "higher_is_better": true
40
+ },
41
+ {
42
+ "metric": "prompt_level_loose_acc",
43
+ "aggregation": "mean",
44
+ "higher_is_better": true
45
+ },
46
+ {
47
+ "metric": "inst_level_loose_acc",
48
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
49
+ "higher_is_better": true
50
+ }
51
+ ],
52
+ "output_type": "generate_until",
53
+ "generation_kwargs": {
54
+ "until": [],
55
+ "do_sample": false,
56
+ "temperature": 0.0,
57
+ "max_gen_toks": 1280
58
+ },
59
+ "repeats": 1,
60
+ "should_decontaminate": false,
61
+ "metadata": {
62
+ "version": 2.0
63
+ }
64
+ }
65
+ },
66
+ "versions": {
67
+ "ifeval": 2.0
68
+ },
69
+ "n-shot": {
70
+ "ifeval": 0
71
+ },
72
+ "higher_is_better": {
73
+ "ifeval": {
74
+ "prompt_level_strict_acc": true,
75
+ "inst_level_strict_acc": true,
76
+ "prompt_level_loose_acc": true,
77
+ "inst_level_loose_acc": true
78
+ }
79
+ },
80
+ "n-samples": {
81
+ "ifeval": {
82
+ "original": 541,
83
+ "effective": 541
84
+ }
85
+ },
86
+ "config": {
87
+ "model": "local-chat-completions",
88
+ "model_args": "base_url=http://localhost:7860/v1",
89
+ "batch_size": 1,
90
+ "batch_sizes": [],
91
+ "device": null,
92
+ "use_cache": null,
93
+ "limit": null,
94
+ "bootstrap_iters": 100000,
95
+ "gen_kwargs": null,
96
+ "random_seed": 0,
97
+ "numpy_seed": 1234,
98
+ "torch_seed": 1234,
99
+ "fewshot_seed": 1234
100
+ },
101
+ "git_hash": "e39df01c",
102
+ "date": 1717614757.4893348,
103
+ "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen 7 3700X 8-Core Processor\nCPU family: 23\nModel: 113\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 0\nFrequency boost: enabled\nCPU(s) scaling MHz: 76%\nCPU max MHz: 4426.1709\nCPU min MHz: 2200.0000\nBogoMIPS: 7189.76\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization: AMD-V\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 4 MiB (8 instances)\nL3 cache: 32 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; Safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[conda] Could not collect",
104
+ "transformers_version": "4.41.2",
105
+ "upper_git_hash": null,
106
+ "task_hashes": {},
107
+ "model_source": "local-chat-completions",
108
+ "model_name": "",
109
+ "model_name_sanitized": "",
110
+ "system_instruction": null,
111
+ "system_instruction_sha": null,
112
+ "chat_template": null,
113
+ "chat_template_sha": null,
114
+ "start_time": 558013.014254911,
115
+ "end_time": 560658.333346293,
116
+ "total_evaluation_time_seconds": "2645.319091381971"
117
+ }
results/Mistral-7B-Instruct-v0.3-GGUF.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "ifeval": {
4
+ "prompt_level_strict_acc,none": 0.49722735674676527,
5
+ "prompt_level_strict_acc_stderr,none": 0.021516243323548144,
6
+ "inst_level_strict_acc,none": 0.5983213429256595,
7
+ "inst_level_strict_acc_stderr,none": "N/A",
8
+ "prompt_level_loose_acc,none": 0.5397412199630314,
9
+ "prompt_level_loose_acc_stderr,none": 0.021448501434135032,
10
+ "inst_level_loose_acc,none": 0.6402877697841727,
11
+ "inst_level_loose_acc_stderr,none": "N/A",
12
+ "alias": "ifeval"
13
+ }
14
+ },
15
+ "group_subtasks": {
16
+ "ifeval": []
17
+ },
18
+ "configs": {
19
+ "ifeval": {
20
+ "task": "ifeval",
21
+ "dataset_path": "wis-k/instruction-following-eval",
22
+ "test_split": "train",
23
+ "doc_to_text": "prompt",
24
+ "doc_to_target": 0,
25
+ "process_results": "def process_results(doc, results):\n eval_logger.warning(\n \"This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models\"\n )\n\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
26
+ "description": "",
27
+ "target_delimiter": " ",
28
+ "fewshot_delimiter": "\n\n",
29
+ "num_fewshot": 0,
30
+ "metric_list": [
31
+ {
32
+ "metric": "prompt_level_strict_acc",
33
+ "aggregation": "mean",
34
+ "higher_is_better": true
35
+ },
36
+ {
37
+ "metric": "inst_level_strict_acc",
38
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
39
+ "higher_is_better": true
40
+ },
41
+ {
42
+ "metric": "prompt_level_loose_acc",
43
+ "aggregation": "mean",
44
+ "higher_is_better": true
45
+ },
46
+ {
47
+ "metric": "inst_level_loose_acc",
48
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
49
+ "higher_is_better": true
50
+ }
51
+ ],
52
+ "output_type": "generate_until",
53
+ "generation_kwargs": {
54
+ "until": [],
55
+ "do_sample": false,
56
+ "temperature": 0.0,
57
+ "max_gen_toks": 1280
58
+ },
59
+ "repeats": 1,
60
+ "should_decontaminate": false,
61
+ "metadata": {
62
+ "version": 2.0
63
+ }
64
+ }
65
+ },
66
+ "versions": {
67
+ "ifeval": 2.0
68
+ },
69
+ "n-shot": {
70
+ "ifeval": 0
71
+ },
72
+ "higher_is_better": {
73
+ "ifeval": {
74
+ "prompt_level_strict_acc": true,
75
+ "inst_level_strict_acc": true,
76
+ "prompt_level_loose_acc": true,
77
+ "inst_level_loose_acc": true
78
+ }
79
+ },
80
+ "n-samples": {
81
+ "ifeval": {
82
+ "original": 541,
83
+ "effective": 541
84
+ }
85
+ },
86
+ "config": {
87
+ "model": "local-chat-completions",
88
+ "model_args": "base_url=http://localhost:7860/v1",
89
+ "batch_size": 1,
90
+ "batch_sizes": [],
91
+ "device": null,
92
+ "use_cache": null,
93
+ "limit": null,
94
+ "bootstrap_iters": 100000,
95
+ "gen_kwargs": null,
96
+ "random_seed": 0,
97
+ "numpy_seed": 1234,
98
+ "torch_seed": 1234,
99
+ "fewshot_seed": 1234
100
+ },
101
+ "git_hash": "e39df01c",
102
+ "date": 1717650829.734338,
103
+ "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen 7 3700X 8-Core Processor\nCPU family: 23\nModel: 113\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 0\nFrequency boost: enabled\nCPU(s) scaling MHz: 84%\nCPU max MHz: 4426.1709\nCPU min MHz: 2200.0000\nBogoMIPS: 7189.76\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization: AMD-V\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 4 MiB (8 instances)\nL3 cache: 32 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; Safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[conda] Could not collect",
104
+ "transformers_version": "4.41.2",
105
+ "upper_git_hash": null,
106
+ "task_hashes": {},
107
+ "model_source": "local-chat-completions",
108
+ "model_name": "",
109
+ "model_name_sanitized": "",
110
+ "system_instruction": null,
111
+ "system_instruction_sha": null,
112
+ "chat_template": null,
113
+ "chat_template_sha": null,
114
+ "start_time": 594085.328143385,
115
+ "end_time": 597056.713585419,
116
+ "total_evaluation_time_seconds": "2971.385442033992"
117
+ }
results/Phi-3-medium-4k-instruct-GGUF.json ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "results": {
3
+ "ifeval": {
4
+ "prompt_level_strict_acc,none": 0.609981515711645,
5
+ "prompt_level_strict_acc_stderr,none": 0.020989594697345435,
6
+ "inst_level_strict_acc,none": 0.7014388489208633,
7
+ "inst_level_strict_acc_stderr,none": "N/A",
8
+ "prompt_level_loose_acc,none": 0.6321626617375231,
9
+ "prompt_level_loose_acc_stderr,none": 0.02075130655602968,
10
+ "inst_level_loose_acc,none": 0.7254196642685852,
11
+ "inst_level_loose_acc_stderr,none": "N/A",
12
+ "alias": "ifeval"
13
+ }
14
+ },
15
+ "group_subtasks": {
16
+ "ifeval": []
17
+ },
18
+ "configs": {
19
+ "ifeval": {
20
+ "task": "ifeval",
21
+ "dataset_path": "wis-k/instruction-following-eval",
22
+ "test_split": "train",
23
+ "doc_to_text": "prompt",
24
+ "doc_to_target": 0,
25
+ "process_results": "def process_results(doc, results):\n eval_logger.warning(\n \"This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models\"\n )\n\n inp = InputExample(\n key=doc[\"key\"],\n instruction_id_list=doc[\"instruction_id_list\"],\n prompt=doc[\"prompt\"],\n kwargs=doc[\"kwargs\"],\n )\n response = results[0]\n\n out_strict = test_instruction_following_strict(inp, response)\n out_loose = test_instruction_following_loose(inp, response)\n\n return {\n \"prompt_level_strict_acc\": out_strict.follow_all_instructions,\n \"inst_level_strict_acc\": out_strict.follow_instruction_list,\n \"prompt_level_loose_acc\": out_loose.follow_all_instructions,\n \"inst_level_loose_acc\": out_loose.follow_instruction_list,\n }\n",
26
+ "description": "",
27
+ "target_delimiter": " ",
28
+ "fewshot_delimiter": "\n\n",
29
+ "num_fewshot": 0,
30
+ "metric_list": [
31
+ {
32
+ "metric": "prompt_level_strict_acc",
33
+ "aggregation": "mean",
34
+ "higher_is_better": true
35
+ },
36
+ {
37
+ "metric": "inst_level_strict_acc",
38
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
39
+ "higher_is_better": true
40
+ },
41
+ {
42
+ "metric": "prompt_level_loose_acc",
43
+ "aggregation": "mean",
44
+ "higher_is_better": true
45
+ },
46
+ {
47
+ "metric": "inst_level_loose_acc",
48
+ "aggregation": "def agg_inst_level_acc(items):\n flat_items = [item for sublist in items for item in sublist]\n inst_level_acc = sum(flat_items) / len(flat_items)\n return inst_level_acc\n",
49
+ "higher_is_better": true
50
+ }
51
+ ],
52
+ "output_type": "generate_until",
53
+ "generation_kwargs": {
54
+ "until": [],
55
+ "do_sample": false,
56
+ "temperature": 0.0,
57
+ "max_gen_toks": 1280
58
+ },
59
+ "repeats": 1,
60
+ "should_decontaminate": false,
61
+ "metadata": {
62
+ "version": 2.0
63
+ }
64
+ }
65
+ },
66
+ "versions": {
67
+ "ifeval": 2.0
68
+ },
69
+ "n-shot": {
70
+ "ifeval": 0
71
+ },
72
+ "higher_is_better": {
73
+ "ifeval": {
74
+ "prompt_level_strict_acc": true,
75
+ "inst_level_strict_acc": true,
76
+ "prompt_level_loose_acc": true,
77
+ "inst_level_loose_acc": true
78
+ }
79
+ },
80
+ "n-samples": {
81
+ "ifeval": {
82
+ "original": 541,
83
+ "effective": 541
84
+ }
85
+ },
86
+ "config": {
87
+ "model": "local-chat-completions",
88
+ "model_args": "base_url=http://localhost:7860/v1",
89
+ "batch_size": 1,
90
+ "batch_sizes": [],
91
+ "device": null,
92
+ "use_cache": null,
93
+ "limit": null,
94
+ "bootstrap_iters": 100000,
95
+ "gen_kwargs": null,
96
+ "random_seed": 0,
97
+ "numpy_seed": 1234,
98
+ "torch_seed": 1234,
99
+ "fewshot_seed": 1234
100
+ },
101
+ "git_hash": "e39df01c",
102
+ "date": 1717656174.5773952,
103
+ "pretty_env_info": "PyTorch version: 2.3.1+cu121\nIs debug build: False\nCUDA used to build PyTorch: 12.1\nROCM used to build PyTorch: N/A\n\nOS: Arch Linux (x86_64)\nGCC version: (GCC) 14.1.1 20240522\nClang version: Could not collect\nCMake version: version 3.29.3\nLibc version: glibc-2.39\n\nPython version: 3.12.3 (main, Apr 23 2024, 09:16:07) [GCC 13.2.1 20240417] (64-bit runtime)\nPython platform: Linux-6.9.2-arch1-1-x86_64-with-glibc2.39\nIs CUDA available: False\nCUDA runtime version: No CUDA\nCUDA_MODULE_LOADING set to: N/A\nGPU models and configuration: No CUDA\nNvidia driver version: No CUDA\ncuDNN version: No CUDA\nHIP runtime version: N/A\nMIOpen runtime version: N/A\nIs XNNPACK available: True\n\nCPU:\nArchitecture: x86_64\nCPU op-mode(s): 32-bit, 64-bit\nAddress sizes: 43 bits physical, 48 bits virtual\nByte Order: Little Endian\nCPU(s): 16\nOn-line CPU(s) list: 0-15\nVendor ID: AuthenticAMD\nModel name: AMD Ryzen 7 3700X 8-Core Processor\nCPU family: 23\nModel: 113\nThread(s) per core: 2\nCore(s) per socket: 8\nSocket(s): 1\nStepping: 0\nFrequency boost: enabled\nCPU(s) scaling MHz: 77%\nCPU max MHz: 4426.1709\nCPU min MHz: 2200.0000\nBogoMIPS: 7189.76\nFlags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ht syscall nx mmxext fxsr_opt pdpe1gb rdtscp lm constant_tsc rep_good nopl xtopology nonstop_tsc cpuid extd_apicid aperfmperf rapl pni pclmulqdq monitor ssse3 fma cx16 sse4_1 sse4_2 x2apic movbe popcnt aes xsave avx f16c rdrand lahf_lm cmp_legacy svm extapic cr8_legacy abm sse4a misalignsse 3dnowprefetch osvw ibs skinit wdt tce topoext perfctr_core perfctr_nb bpext perfctr_llc mwaitx cpb cat_l3 cdp_l3 hw_pstate ssbd mba ibpb stibp vmmcall fsgsbase bmi1 avx2 smep bmi2 cqm rdt_a rdseed adx smap clflushopt clwb sha_ni xsaveopt xsavec xgetbv1 cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local clzero irperf xsaveerptr rdpru wbnoinvd arat npt lbrv svm_lock nrip_save tsc_scale vmcb_clean flushbyasid decodeassists pausefilter pfthreshold avic v_vmsave_vmload vgif v_spec_ctrl umip rdpid overflow_recov succor smca sev sev_es\nVirtualization: AMD-V\nL1d cache: 256 KiB (8 instances)\nL1i cache: 256 KiB (8 instances)\nL2 cache: 4 MiB (8 instances)\nL3 cache: 32 MiB (2 instances)\nNUMA node(s): 1\nNUMA node0 CPU(s): 0-15\nVulnerability Gather data sampling: Not affected\nVulnerability Itlb multihit: Not affected\nVulnerability L1tf: Not affected\nVulnerability Mds: Not affected\nVulnerability Meltdown: Not affected\nVulnerability Mmio stale data: Not affected\nVulnerability Reg file data sampling: Not affected\nVulnerability Retbleed: Mitigation; untrained return thunk; SMT enabled with STIBP protection\nVulnerability Spec rstack overflow: Mitigation; Safe RET\nVulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl\nVulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization\nVulnerability Spectre v2: Mitigation; Retpolines; IBPB conditional; STIBP always-on; RSB filling; PBRSB-eIBRS Not affected; BHI Not affected\nVulnerability Srbds: Not affected\nVulnerability Tsx async abort: Not affected\n\nVersions of relevant libraries:\n[pip3] numpy==1.26.4\n[pip3] torch==2.3.1\n[conda] Could not collect",
104
+ "transformers_version": "4.41.2",
105
+ "upper_git_hash": null,
106
+ "task_hashes": {},
107
+ "model_source": "local-chat-completions",
108
+ "model_name": "",
109
+ "model_name_sanitized": "",
110
+ "system_instruction": null,
111
+ "system_instruction_sha": null,
112
+ "chat_template": null,
113
+ "chat_template_sha": null,
114
+ "start_time": 599429.903058813,
115
+ "end_time": 604622.355146192,
116
+ "total_evaluation_time_seconds": "5192.452087379061"
117
+ }