Update: Add Verilator as a new sim

app.py

@@ -171,8 +171,14 @@ function refresh() {
 with gr.Blocks(
     css=custom_css, js=js_func, theme=gr.themes.Default(primary_hue=colors.emerald)
 ) as app:
-    df, benchmarks, metrics, default_metric = read_data()
-    df_agg = parse_agg("./results/aggregated_scores.csv")
+    df_icarus, benchmarks, metrics, default_metric = read_data(
+        "results/results_icarus.json"
+    )
+    df_agg_icarus = parse_agg("results/aggregated_scores_icarus.csv")
+    df_verilator, _, _, _ = read_data("results/results_verilator.json")
+    df_agg_verilator = parse_agg("results/aggregated_scores_verilator.csv")
+    df = df_icarus
+    df_agg = df_agg_icarus
     tasks = ["Spec-to-RTL", "Code Completion", "Line Completion"]
     s2r_benchs = ["VerilogEval S2R", "RTLLM"]
     cc_benchs = ["VerilogEval MC", "VeriGen"]
@@ -225,27 +231,35 @@ with gr.Blocks(
     )
     gr.HTML(
         """
-    <div style=" margin-top:-10px !important;">
-        <p style="margin-bottom: 15px;  text-align: start !important;">Welcome to the TuRTLe Model Leaderboard! TuRTLe is a <b>unified evaluation framework designed to systematically assess Large Language Models (LLMs) in RTL (Register-Transfer Level) generation</b> for hardware design.
-        Evaluation criteria include <b>syntax correctness, functional accuracy, synthesizability, and post-synthesis quality</b> (PPA: Power, Performance, Area). TuRTLe integrates multiple benchmarks to highlight strengths and weaknesses of available LLMs. 
-        Use the filters below to explore different RTL benchmarks and models.</p>
-    <p style="margin-top: 15px; text-align: start !important; "><span style="font-variant: small-caps; font-weight: bold;">NEW UPDATE (JUNE 2025)</span>: We make our framework open-source on GitHub and we add 7 new recent models! For a total of 40 base and instruct models and 5 RTL benchmarks.</p>
-    </div>
-    """
+        <div style=" margin-top:-10px !important;">
+            <p style="margin-bottom: 15px;  text-align: start !important;">Welcome to the TuRTLe Model Leaderboard! TuRTLe is a <b>unified evaluation framework designed to systematically assess Large Language Models (LLMs) in RTL (Register-Transfer Level) generation</b> for hardware design.
+            Evaluation criteria include <b>syntax correctness, functional accuracy, synthesizability, and post-synthesis quality</b> (PPA: Power, Performance, Area). TuRTLe integrates multiple benchmarks to highlight strengths and weaknesses of available LLMs. 
+            Use the filters below to explore different RTL benchmarks, simulators and models.</p>
+            <p style="margin-top:10px; text-align:start !important;"> <span style="font-variant:small-caps; font-weight:bold;">UPDATE (JULY 2025)</span>: We have added Verilator as a second simulator, the original Icarus data is still available. You can now filter the data by simulator</p>
+            <p style="margin-top: -6px; text-align: start !important; "><span style="font-variant: small-caps; font-weight: bold;">UPDATE (JUNE 2025)</span>: We make our framework open-source on GitHub and we add 7 new recent models! For a total of 40 base and instruct models and 5 RTL benchmarks</p>
+        </div>
+        """
     )
     with gr.Tabs():
         with gr.Tab("Leaderboard"):
             with gr.Row(equal_height=True):
-                with gr.Column():
+                with gr.Column(scale=4):
                     task_radio = gr.Radio(
                         choices=tasks, label="Select Task", value="Spec-to-RTL"
                     )
-                with gr.Column():
+                with gr.Column(scale=3):
                     benchmark_radio = gr.Radio(
                         choices=["All"] + s2r_benchs,
                         label="Select Benchmark",
                         value="All",
                     )
+                with gr.Column(scale=2, min_width=180):
+                    simulator_radio = gr.Radio(
+                        choices=["Icarus", "Verilator"],
+                        value="Icarus",
+                        label="Simulator",
+                        scale=1,
+                    )
 
             with gr.Row(equal_height=True):
                 search_box = gr.Textbox(
@@ -447,6 +461,28 @@ with gr.Blocks(
         fig = generate_scatter_plot(benchmark, metric)
         return gr.update(value=benchmark), fig
 
+    def on_simulator_change(
+        simulator,
+        task,
+        benchmark,
+        model_type,
+        search,
+        max_params,
+        plot_bench,
+        plot_metric,
+    ):
+        global df, df_agg
+        if simulator == "Icarus":
+            df, df_agg = df_icarus, df_agg_icarus
+        else:
+            df, df_agg = df_verilator, df_agg_verilator
+
+        leaderboard_df = filter_leaderboard(
+            task, benchmark, model_type, search, max_params
+        )
+        fig = generate_scatter_plot(plot_bench, plot_metric)
+        return leaderboard_df, fig
+
     bubble_benchmark.change(
         fn=on_benchmark_change,
         inputs=[bubble_benchmark, bubble_metric],
@@ -481,6 +517,21 @@ with gr.Blocks(
         """,
     )
 
+    simulator_radio.change(
+        fn=on_simulator_change,
+        inputs=[
+            simulator_radio,
+            task_radio,
+            benchmark_radio,
+            model_type_dropdown,
+            search_box,
+            params_slider,
+            bubble_benchmark,
+            bubble_metric,
+        ],
+        outputs=[leaderboard, scatter_plot],
+    )
+
 
 app.launch(
     allowed_paths=[

results/aggregated_scores.csv

@@ -1,28 +0,0 @@
-Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
-DeepSeek R1,75.53,72.96,77.67,77.55,68.49,57.82
-Llama 3.1 405B,53.23,53.88,56.55,54.35,42.26,52.35
-Qwen3 236B A22B,69.16,63.42,74.83,68.36,50.48,47.15
-Llama 3.(1-3) 70B,39.48,43.29,39.47,40.83,39.53,51.42
-Qwen2.5 72B,49.36,47.23,50.22,50.74,46.51,35.65
-QwQ 32B,62.6,39.46,65.02,38.68,54.6,42.03
-Qwen2.5 32B,50.39,38.93,50.86,41.01,48.86,32.09
-StarChat2 15B v0.1,38.76,38.98,36.68,35.58,45.61,50.2
-DeepSeek R1 Distill Qwen 14B,23.14,23.3,24.94,24.3,17.22,20.01
-CodeLlama 70B,33.04,32.86,32.2,32.27,35.81,34.8
-DeepSeek Coder 33B,27.03,36.31,22.65,37.64,41.47,31.91
-QwenCoder 2.5 32B,44.02,43.75,43.68,44.05,45.15,42.76
-QwenCoder 2.5 14B,37.69,38.97,35.32,40.26,45.5,34.72
-DeepCoder 14B,26.4,30.75,27.32,33.18,23.38,22.75
-OpenCoder 8B,30.06,35.86,26.8,34.57,40.83,40.1
-SeedCoder 8B,50.89,34.05,51.71,36.52,48.2,25.89
-SeedCoder 8B Reasoning,43.75,47.1,47.85,46.76,30.22,48.23
-QwenCoder 2.5 7B,14.15,32.86,6.57,33.0,39.16,32.4
-"DeepSeek Coder 6,7B",31.87,27.89,28.88,28.99,41.75,24.27
-RTLCoder Mistral,21.82,28.65,23.71,26.34,15.58,36.27
-RTLCoder DeepSeek,37.22,36.64,38.33,36.13,33.58,38.33
-OriGen,52.88,51.89,53.0,50.0,52.47,58.12
-CodeV R1 Distill Qwen 7B,36.12,26.84,32.35,20.56,48.57,47.55
-HaVen-CodeQwen,43.58,47.13,44.67,47.23,39.98,46.8
-CodeV-CL-7B,14.73,33.73,12.71,30.93,21.38,42.97
-CodeV-QW-7B,20.37,50.11,18.82,50.64,25.48,48.38
-CodeV-DS-6.7B,19.62,47.1,14.8,46.08,35.52,50.46

results/aggregated_scores_icarus.csv

@@ -0,0 +1,29 @@
+Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
+DeepSeek R1-0528,76.79,77.79,78.84,79.65,70.04,71.64
+DeepSeek R1,75.53,76.24,77.67,77.55,68.49,71.92
+Llama 3.1 405B,53.23,55.25,56.55,54.35,42.26,58.23
+Qwen3 235B A22B,69.16,66.42,74.83,68.36,50.48,60.01
+Qwen2.5 72B,49.36,49.97,50.22,50.74,46.51,47.42
+Llama 3.(1-3) 70B,39.48,46.29,39.47,40.83,39.53,64.29
+QwQ 32B,62.6,41.92,65.02,38.68,54.6,52.6
+Qwen2.5 32B,50.39,41.94,50.86,41.01,48.86,45.0
+StarChat2 15B v0.1,38.76,41.72,36.68,35.58,45.61,61.96
+DeepSeek R1 Distill Qwen 14B,23.14,24.94,24.94,24.3,17.22,27.07
+CodeLlama 70B,33.04,35.59,32.2,32.27,35.81,46.56
+DeepSeek Coder 33B,27.03,39.59,22.65,37.64,41.47,46.01
+QwenCoder 2.5 32B,44.02,46.49,43.68,44.05,45.15,54.52
+DeepCoder 14B,26.4,32.92,27.32,33.18,23.38,32.07
+QwenCoder 2.5 14B,37.69,41.98,35.32,40.26,45.5,47.64
+SeedCoder 8B,50.89,36.79,51.71,36.52,48.2,37.66
+SeedCoder 8B Reasoning,43.75,49.83,47.85,46.76,30.22,59.97
+OpenCoder 8B,30.06,36.95,26.8,34.57,40.83,44.8
+QwenCoder 2.5 7B,14.15,34.5,6.57,33.0,39.16,39.46
+DeepSeek Coder 6.7B,31.87,30.9,28.88,28.99,41.75,37.19
+CodeV R1 Distill Qwen 7B,36.12,30.66,32.35,20.56,48.57,63.98
+HaVen-CodeQwen,43.58,50.68,44.67,47.23,39.98,62.04
+CodeV-QW-7B,20.37,52.85,18.82,50.64,25.48,60.14
+RTLCoder Mistral,21.82,32.19,23.71,26.34,15.58,51.5
+RTLCoder DeepSeek,37.22,40.19,38.33,36.13,33.58,53.57
+OriGen,52.88,55.97,53.0,50.0,52.47,75.66
+CodeV-CL-7B,14.73,35.1,12.71,30.93,21.38,48.85
+CodeV-DS-6.7B,19.62,49.29,14.8,46.08,35.52,59.87

results/aggregated_scores_verilator.csv

@@ -0,0 +1,29 @@
+Model,Agg S2R,Agg MC,Agg VerilogEval S2R,Agg VerilogEval MC,Agg RTLLM,Agg VeriGen
+DeepSeek R1-0528,75.83,77.12,77.64,78.78,69.85,71.64
+DeepSeek R1,75.78,75.4,78.04,76.42,68.31,72.05
+Llama 3.1 405B,52.08,55.01,54.39,55.1,44.48,54.7
+Qwen3 236B A22B,69.17,65.8,74.24,67.55,52.44,60.01
+Qwen2.5 72B,51.72,52.09,52.7,52.44,48.49,50.95
+Llama 3.(1-3) 70B,40.06,47.44,39.84,42.69,40.79,63.12
+QwQ 32B,63.76,44.52,66.09,38.64,56.06,63.91
+Qwen2.5 32B,53.2,44.35,54.23,43.05,49.81,48.64
+StarChat2 15B v0.1,40.2,41.91,37.97,35.85,47.54,61.91
+DeepSeek R1 Distill Qwen 14B,22.93,25.34,24.44,24.43,17.96,28.33
+CodeLlama 70B,34.55,38.02,33.58,33.89,37.75,51.62
+DeepSeek Coder 33B,27.93,41.16,23.14,38.14,43.71,51.12
+QwenCoder 2.5 32B,45.72,48.38,45.31,46.49,47.07,54.62
+DeepCoder 14B,27.06,33.64,27.87,33.39,24.38,34.45
+QwenCoder 2.5 14B,39.39,43.55,36.86,41.54,47.73,50.17
+SeedCoder 8B,52.04,38.31,52.58,37.24,50.25,41.84
+SeedCoder 8B Reasoning,43.41,50.73,47.02,47.51,31.51,61.35
+OpenCoder 8B,30.45,37.27,26.96,35.34,41.96,43.63
+QwenCoder 2.5 7B,14.91,37.2,7.08,34.02,40.74,47.7
+DeepSeek Coder 6.7B,34.14,31.2,30.46,29.39,46.28,37.19
+CodeV R1 Distill Qwen 7B,37.26,30.97,33.33,21.67,50.21,61.63
+HaVen-CodeQwen,44.57,52.4,45.09,48.07,42.87,66.7
+CodeV-QW-7B,21.69,52.12,19.85,50.51,27.77,57.45
+RTLCoder Mistral,22.64,33.32,24.75,26.44,15.68,56.02
+RTLCoder DeepSeek,38.48,39.89,39.19,35.74,36.14,53.57
+OriGen,52.85,55.3,52.72,49.84,53.29,73.3
+CodeV-CL-7B,15.32,35.2,13.21,31.06,22.28,48.85
+CodeV-DS-6.7B,19.28,49.43,14.23,45.91,35.94,61.05

results/parse.py

@@ -111,7 +111,7 @@ model_details = {
         "Coding",
         "V1",
     ),
-    "DeepSeek Coder 6,7B": (
+    "DeepSeek Coder 6.7B": (
         "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-instruct",
         6.74,
         "Coding",
@@ -198,6 +198,7 @@ def parse_results(csv_path: str) -> list[dict]:
         metrics, benchs = get_headers(reader)
         for i, row in enumerate(reader):
             model = row[0]
+            print(model)
             url, params, type, release = get_model_params_and_url(model)
             models.append(model)
             row = row[1:]
@@ -220,12 +221,12 @@ def parse_results(csv_path: str) -> list[dict]:
     return dataset
 
 
-def parse_agg(csv_path: str) -> list[dict]:
+def parse_agg(csv_path: str = "results/aggregated_scores_icarus.csv") -> pd.DataFrame:
     """
     Each row has the following format:
         MODEL | BENCHMARK | TASK | METRIC | RESULT
     """
-    return pd.read_csv("results/aggregated_scores.csv")
+    return pd.read_csv(csv_path)
 
 
 def writeJson(data: list):
@@ -234,15 +235,16 @@ def writeJson(data: list):
     print("Done")
 
 
-def read_json():
-    json_path = "results/results.json"
+def read_json(json_path: str = "results/results_icarus.json"):
     with open(json_path, "r", encoding="utf-8") as file:
         data = json.load(file)
     return data
 
 
-def read_data() -> Union[pd.DataFrame, list, list, str]:
-    data = read_json()
+def read_data(
+    json_path: str = "results/results_icarus.json",
+) -> tuple[pd.DataFrame, list, list, str]:
+    data = read_json(json_path)
     df = pd.DataFrame(data)
     df.rename(
         columns={

results/results.csv

@@ -1,30 +0,0 @@
-,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
-,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
-DeepSeek R1-0528,96.15,86.12,81.54,64.9,81.28,64.49,79.15,73.33,78.09,65.64,79.28,71.15,-1,95.38,91.76,81.54,62.35,81.41,62.35,80.27,52.12,78.69,62.25,79.99,65.32
-DeepSeek R1,97.18,89.80,79.74,65.71,79.62,63.27,78.33,71.34,76.49,64.06,78.19,70.08,-1.00,97.44,96.47,79.49,60.00,79.49,60.00,78.27,50.25,76.43,60.15,77.96,63.07
-Llama 3.1 405B,87.44,77.14,58.97,45.71,58.85,41.63,57.58,50.88,55.93,32.44,56.13,43.45,34.62,88.59,95.29,56.15,52.94,55.90,52.94,55.13,49.22,53.45,52.52,54.48,55.31
-Qwen3 236B A22B,91.28,73.88,76.92,51.43,76.79,48.57,75.25,54.61,73.56,46.37,75.67,50.47,41.94,82.18,87.06,69.62,49.41,69.62,49.41,69.04,41.82,66.89,49.64,69.15,49.99
-Llama 3.(1-3) 70B,66.15,73.88,40.64,42.45,40.64,39.18,40.46,40.81,38.08,38.14,39.86,39.65,28.72,84.74,89.41,41.67,51.76,41.67,51.76,41.38,50.61,39.75,51.76,41.36,51.88
-Qwen2.5 72B,82.18,79.59,52.44,45.31,51.92,44.08,51.83,46.47,48.75,45.40,50.09,47.65,37.44,80.90,84.71,52.95,35.29,52.69,35.29,51.66,35.82,49.37,35.20,51.18,35.94
-QwQ 32B,87.95,82.45,66.41,56.73,66.41,52.24,66.15,55.83,63.80,51.91,65.12,56.07,-1.00,58.97,68.24,40.00,42.35,39.62,42.35,39.40,40.90,37.53,42.31,39.10,42.87
-Qwen2.5 32B,88.59,84.08,52.56,50.20,52.18,46.12,52.32,49.73,49.43,46.43,50.82,50.43,28.93,93.21,85.88,41.54,32.94,41.54,32.94,41.31,30.65,40.48,33.11,41.23,32.50
-StarChat2 15B v0.1,88.46,84.90,37.95,44.49,37.95,44.08,37.56,46.95,35.30,43.22,37.19,46.65,13.42,79.74,92.94,36.41,51.76,36.03,51.76,36.08,46.30,34.91,51.49,35.76,52.80
-DeepSeek R1 Distill Qwen 14B,42.18,34.69,25.51,18.37,25.51,16.33,25.36,17.86,24.19,16.48,25.27,17.33,-1.00,45.00,44.71,25.64,21.18,25.26,21.18,24.79,17.65,23.48,21.08,24.63,21.29
-CodeLlama 70B,67.05,69.80,33.08,36.33,33.08,34.29,32.69,37.19,31.46,34.29,32.44,35.95,24.33,90.77,88.24,33.33,35.29,33.33,35.29,33.02,34.03,30.80,35.15,32.99,35.21
-DeepSeek Coder 33B,62.82,83.67,23.33,42.45,23.08,42.04,22.86,42.29,22.81,39.42,22.29,42.71,24.58,75.26,88.24,39.62,31.76,39.36,31.76,38.23,32.16,36.79,31.46,37.90,32.12
-QwenCoder 2.5 32B,87.18,77.96,45.00,43.27,44.87,43.27,44.25,46.82,43.03,43.20,43.76,45.42,31.07,83.72,87.06,45.64,42.35,45.13,42.35,44.59,42.79,43.01,42.24,44.55,43.25
-QwenCoder 2.5 14B,78.97,81.63,37.82,46.12,37.44,45.31,35.94,45.82,34.83,44.64,35.18,46.05,37.53,80.00,83.53,41.67,35.29,41.15,35.29,40.74,34.17,39.20,35.32,40.83,34.67
-DeepCoder 14B,43.85,39.59,28.08,23.67,28.08,22.04,27.94,25.00,26.26,22.00,27.77,23.15,-1.00,61.92,48.24,34.10,23.53,33.72,23.53,33.70,21.18,32.17,23.43,33.67,23.65
-OpenCoder 8B,78.21,75.92,28.46,42.86,27.82,40.82,27.34,41.36,25.95,39.77,27.11,41.36,16.17,80.00,95.29,35.64,41.18,35.38,41.18,35.12,37.69,33.47,41.05,35.13,41.55
-SeedCoder 8B,91.41,85.31,53.46,47.35,53.33,46.53,52.86,49.42,50.62,45.60,51.65,49.59,28.23,77.44,94.12,37.31,30.59,37.31,27.06,37.32,23.53,35.35,26.92,36.89,27.23
-SeedCoder 8B Reasoning,67.82,53.47,49.23,30.20,49.23,29.39,48.92,32.04,46.76,28.64,47.87,29.99,-1.00,83.33,78.82,48.21,50.59,48.08,50.59,47.78,41.74,45.44,50.02,47.06,52.92
-QwenCoder 2.5 7B,20.13,76.33,6.92,38.78,6.67,37.14,6.51,40.65,6.63,37.25,6.56,39.58,28.33,74.10,90.59,33.72,32.94,33.72,32.94,33.59,30.67,31.78,33.01,33.62,33.51
-"DeepSeek Coder 6,7B",82.05,78.78,29.62,41.22,29.49,38.78,29.51,42.62,27.73,39.33,29.41,43.30,24.63,67.18,84.71,31.67,24.71,29.87,24.71,29.78,23.53,27.98,24.50,29.21,24.79
-RTLCoder Mistral,54.87,32.24,24.62,16.33,24.62,15.92,24.28,16.03,22.78,14.71,24.06,16.00,14.77,60.51,85.88,27.05,36.47,27.05,36.47,26.94,34.63,25.22,36.55,26.87,37.64
-RTLCoder DeepSeek,84.62,73.06,39.49,37.14,39.49,34.69,38.91,34.30,37.52,32.76,38.55,33.69,19.35,77.31,85.88,36.92,40.00,36.79,40.00,36.94,35.57,34.84,39.83,36.62,39.60
-OriGen,96.15,81.63,54.23,50.61,54.23,50.61,54.29,53.10,51.57,50.86,53.15,53.44,17.07,92.44,98.82,50.77,58.82,50.77,58.82,50.95,54.14,48.53,58.81,50.51,61.40
-CodeV R1 Distill Qwen 7B,56.92,73.06,33.33,49.80,33.33,47.35,32.58,49.25,32.01,47.45,32.45,49.01,-1.00,92.69,89.41,21.28,49.41,21.28,49.41,21.04,43.68,19.59,49.06,21.05,49.91
-HaVen-CodeQwen,93.33,80.41,47.31,42.86,46.15,41.22,45.08,40.59,44.26,38.83,44.68,40.53,25.14,93.59,100.00,50.13,47.06,49.49,47.06,47.55,46.60,47.05,47.14,47.09,46.67
-CodeV-CL-7B,32.18,48.16,13.08,24.49,12.95,21.63,12.80,22.25,12.51,20.59,12.82,21.29,12.27,92.05,98.82,31.79,43.53,31.79,43.53,31.74,42.25,29.45,43.46,31.61,43.20
-CodeV-QW-7B,45.38,68.16,19.62,34.29,18.97,26.53,18.91,28.14,18.71,21.80,18.85,26.50,20.94,93.33,100.00,52.31,48.24,51.54,48.24,51.69,48.14,48.79,48.18,51.45,48.81
-CodeV-DS-6.7B,33.59,67.35,15.00,38.78,15.00,37.14,15.10,35.56,14.46,35.13,14.85,35.88,21.26,95.51,100.00,47.05,50.59,47.05,50.59,47.37,50.47,44.35,50.54,46.52,50.36

results/results_icarus.csv

@@ -0,0 +1,30 @@
+,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
+,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
+DeepSeek R1-0528,96.150000000000006,86.120000000000005,81.540000000000006,64.900000000000006,81.280000000000001,64.489999999999995,79.150000000000006,73.329999999999998,78.090000000000003,65.640000000000001,79.280000000000001,71.150000000000006,-1,95.379999999999995,91.760000000000005,81.540000000000006,74.120000000000005,81.409999999999997,74.120000000000005,80.269999999999996,63.859999999999999,78.689999999999998,74.049999999999997,79.989999999999995,77.010000000000005
+DeepSeek R1,97.180000000000007,89.799999999999997,79.739999999999995,65.709999999999994,79.620000000000005,63.270000000000003,78.329999999999998,71.340000000000003,76.489999999999995,64.060000000000002,78.189999999999998,70.079999999999998,-1,97.439999999999998,96.469999999999999,79.489999999999995,74.120000000000005,79.489999999999995,74.120000000000005,78.269999999999996,64.340000000000003,76.430000000000007,74.290000000000006,77.959999999999994,77.120000000000005
+Llama 3.1 405B,87.439999999999998,77.140000000000001,58.969999999999999,45.710000000000001,58.850000000000001,41.630000000000003,57.579999999999998,50.880000000000003,55.93,32.439999999999998,56.130000000000003,43.450000000000003,34.619999999999997,88.590000000000003,95.290000000000006,56.149999999999999,58.82,55.899999999999999,58.82,55.130000000000003,55.100000000000001,53.450000000000003,58.399999999999999,54.479999999999997,61.200000000000003
+Qwen3 236B A22B,91.280000000000001,73.879999999999995,76.920000000000002,51.43,76.790000000000006,48.57,75.25,54.609999999999999,73.560000000000002,46.369999999999997,75.670000000000002,50.469999999999999,41.939999999999998,82.180000000000007,87.060000000000002,69.620000000000005,62.350000000000001,69.620000000000005,62.350000000000001,69.040000000000006,54.630000000000003,66.890000000000001,62.630000000000003,69.150000000000006,62.780000000000001
+Qwen2.5 72B,82.180000000000007,79.590000000000003,52.439999999999998,45.310000000000002,51.920000000000002,44.079999999999998,51.829999999999998,46.469999999999999,48.75,45.399999999999999,50.090000000000003,47.649999999999999,37.439999999999998,80.900000000000006,84.709999999999994,52.950000000000003,47.060000000000002,52.689999999999998,47.060000000000002,51.659999999999997,47.590000000000003,49.369999999999997,46.960000000000001,51.18,47.700000000000003
+Llama 3.(1-3) 70B,66.150000000000006,73.879999999999995,40.640000000000001,42.450000000000003,40.640000000000001,39.18,40.460000000000001,40.810000000000002,38.079999999999998,38.140000000000001,39.859999999999999,39.649999999999999,28.719999999999999,84.739999999999995,89.409999999999997,41.670000000000002,65.879999999999995,41.670000000000002,64.709999999999994,41.380000000000003,63.469999999999999,39.75,64.689999999999998,41.359999999999999,64.709999999999994
+QwQ 32B,87.950000000000003,82.450000000000003,66.409999999999997,56.729999999999997,66.409999999999997,52.240000000000002,66.150000000000006,55.829999999999998,63.799999999999997,51.909999999999997,65.120000000000005,56.07,-1,58.969999999999999,68.239999999999995,40,52.939999999999998,39.619999999999997,52.939999999999998,39.399999999999999,51.469999999999999,37.530000000000001,52.93,39.100000000000001,53.390000000000001
+Qwen2.5 32B,88.590000000000003,84.079999999999998,52.560000000000002,50.200000000000003,52.18,46.119999999999997,52.32,49.729999999999997,49.43,46.43,50.82,50.43,28.93,93.209999999999994,85.879999999999995,41.539999999999999,45.880000000000003,41.539999999999999,45.880000000000003,41.310000000000002,43.560000000000002,40.479999999999997,46.079999999999998,41.229999999999997,45.369999999999997
+StarChat2 15B v0.1,88.459999999999994,84.900000000000006,37.950000000000003,44.490000000000002,37.950000000000003,44.079999999999998,37.560000000000002,46.950000000000003,35.299999999999997,43.219999999999999,37.189999999999998,46.649999999999999,13.42,79.739999999999995,92.939999999999998,36.409999999999997,63.530000000000001,36.030000000000001,63.530000000000001,36.079999999999998,58.060000000000002,34.909999999999997,63.259999999999998,35.759999999999998,64.560000000000002
+DeepSeek R1 Distill Qwen 14B,42.18,34.689999999999998,25.510000000000002,18.370000000000001,25.510000000000002,16.329999999999998,25.359999999999999,17.859999999999999,24.190000000000001,16.48,25.27,17.329999999999998,-1,45,44.710000000000001,25.640000000000001,28.239999999999998,25.260000000000002,28.239999999999998,24.789999999999999,24.710000000000001,23.48,28.140000000000001,24.629999999999999,28.350000000000001
+CodeLlama 70B,67.049999999999997,69.799999999999997,33.079999999999998,36.329999999999998,33.079999999999998,34.289999999999999,32.689999999999998,37.189999999999998,31.460000000000001,34.289999999999999,32.439999999999998,35.950000000000003,24.329999999999998,90.769999999999996,88.239999999999995,33.329999999999998,47.060000000000002,33.329999999999998,47.060000000000002,33.020000000000003,45.799999999999997,30.800000000000001,46.909999999999997,32.990000000000002,46.979999999999997
+DeepSeek Coder 33B,62.82,83.670000000000002,23.329999999999998,42.450000000000003,23.079999999999998,42.039999999999999,22.859999999999999,42.289999999999999,22.809999999999999,39.420000000000002,22.289999999999999,42.710000000000001,24.579999999999998,75.260000000000005,88.239999999999995,39.619999999999997,45.880000000000003,39.359999999999999,45.880000000000003,38.229999999999997,46.259999999999998,36.789999999999999,45.609999999999999,37.899999999999999,46.170000000000002
+QwenCoder 2.5 32B,87.180000000000007,77.959999999999994,45,43.270000000000003,44.869999999999997,43.270000000000003,44.25,46.82,43.030000000000001,43.200000000000003,43.759999999999998,45.420000000000002,31.07,83.719999999999999,87.060000000000002,45.640000000000001,54.119999999999997,45.130000000000003,54.119999999999997,44.590000000000003,54.549999999999997,43.009999999999998,54.009999999999998,44.549999999999997,55.009999999999998
+DeepCoder 14B,43.850000000000001,39.590000000000003,28.079999999999998,23.670000000000002,28.079999999999998,22.039999999999999,27.940000000000001,25,26.260000000000002,22,27.77,23.149999999999999,-1,61.920000000000002,48.240000000000002,34.100000000000001,32.939999999999998,33.719999999999999,32.939999999999998,33.700000000000003,30.469999999999999,32.170000000000002,32.840000000000003,33.670000000000002,32.909999999999997
+QwenCoder 2.5 14B,78.969999999999999,81.629999999999995,37.82,46.119999999999997,37.439999999999998,45.310000000000002,35.939999999999998,45.82,34.829999999999998,44.640000000000001,35.18,46.049999999999997,37.530000000000001,80,83.530000000000001,41.670000000000002,48.240000000000002,41.149999999999999,48.240000000000002,40.740000000000002,47.090000000000003,39.200000000000003,48.289999999999999,40.829999999999998,47.539999999999999
+SeedCoder 8B,91.409999999999997,85.310000000000002,53.460000000000001,47.350000000000001,53.329999999999998,46.530000000000001,52.859999999999999,49.420000000000002,50.619999999999997,45.600000000000001,51.649999999999999,49.590000000000003,28.23,77.439999999999998,94.120000000000005,37.310000000000002,42.350000000000001,37.310000000000002,38.82,37.32,35.289999999999999,35.350000000000001,38.689999999999998,36.890000000000001,38.990000000000002
+SeedCoder 8B Reasoning,67.819999999999993,53.469999999999999,49.229999999999997,30.199999999999999,49.229999999999997,29.390000000000001,48.920000000000002,32.039999999999999,46.759999999999998,28.640000000000001,47.869999999999997,29.989999999999998,-1,83.329999999999998,78.819999999999993,48.210000000000001,62.350000000000001,48.079999999999998,62.350000000000001,47.780000000000001,53.479999999999997,45.439999999999998,61.810000000000002,47.060000000000002,64.620000000000005
+OpenCoder 8B,78.209999999999994,75.920000000000002,28.460000000000001,42.859999999999999,27.82,40.82,27.34,41.359999999999999,25.949999999999999,39.770000000000003,27.109999999999999,41.359999999999999,16.170000000000002,80,95.290000000000006,35.640000000000001,45.880000000000003,35.380000000000003,45.880000000000003,35.119999999999997,42.399999999999999,33.469999999999999,45.75,35.130000000000003,46.259999999999998
+QwenCoder 2.5 7B,20.129999999999999,76.329999999999998,6.9199999999999999,38.780000000000001,6.6699999999999999,37.140000000000001,6.5099999999999998,40.649999999999999,6.6299999999999999,37.25,6.5599999999999996,39.579999999999998,28.329999999999998,74.099999999999994,90.590000000000003,33.719999999999999,40,33.719999999999999,40,33.590000000000003,37.729999999999997,31.780000000000001,40.07,33.619999999999997,40.57
+DeepSeek Coder 6.7B,82.049999999999997,78.780000000000001,29.620000000000001,41.219999999999999,29.489999999999998,38.780000000000001,29.510000000000002,42.619999999999997,27.73,39.329999999999998,29.41,43.299999999999997,24.629999999999999,67.180000000000007,84.709999999999994,31.670000000000002,37.649999999999999,29.870000000000001,37.649999999999999,29.780000000000001,36.450000000000003,27.98,37.469999999999999,29.210000000000001,37.659999999999997
+CodeV R1 Distill Qwen 7B,56.920000000000002,73.060000000000002,33.329999999999998,49.799999999999997,33.329999999999998,47.350000000000001,32.579999999999998,49.25,32.009999999999998,47.450000000000003,32.450000000000003,49.009999999999998,-1,92.689999999999998,89.409999999999997,21.280000000000001,65.879999999999995,21.280000000000001,65.879999999999995,21.039999999999999,60.100000000000001,19.59,65.590000000000003,21.050000000000001,66.239999999999995
+HaVen-CodeQwen,93.329999999999998,80.409999999999997,47.310000000000002,42.859999999999999,46.149999999999999,41.219999999999999,45.079999999999998,40.590000000000003,44.259999999999998,38.829999999999998,44.68,40.530000000000001,25.140000000000001,93.590000000000003,100,50.130000000000003,62.350000000000001,49.490000000000002,62.350000000000001,47.549999999999997,61.82,47.049999999999997,62.530000000000001,47.090000000000003,61.759999999999998
+CodeV-QW-7B,45.380000000000003,68.159999999999997,19.620000000000001,34.289999999999999,18.969999999999999,26.530000000000001,18.91,28.140000000000001,18.710000000000001,21.800000000000001,18.850000000000001,26.5,20.940000000000001,93.329999999999998,100,52.310000000000002,60,51.539999999999999,60,51.689999999999998,59.899999999999999,48.789999999999999,59.950000000000003,51.450000000000003,60.579999999999998
+RTLCoder Mistral,54.869999999999997,32.240000000000002,24.620000000000001,16.329999999999998,24.620000000000001,15.92,24.280000000000001,16.030000000000001,22.780000000000001,14.710000000000001,24.059999999999999,16,14.77,60.509999999999998,85.879999999999995,27.050000000000001,51.759999999999998,27.050000000000001,51.759999999999998,26.940000000000001,49.850000000000001,25.219999999999999,51.939999999999998,26.870000000000001,52.719999999999999
+RTLCoder DeepSeek,84.620000000000005,73.060000000000002,39.490000000000002,37.140000000000001,39.490000000000002,34.689999999999998,38.909999999999997,34.299999999999997,37.520000000000003,32.759999999999998,38.549999999999997,33.689999999999998,19.350000000000001,77.310000000000002,85.879999999999995,36.920000000000002,55.289999999999999,36.789999999999999,55.289999999999999,36.939999999999998,50.789999999999999,34.840000000000003,55.219999999999999,36.619999999999997,54.689999999999998
+OriGen,96.150000000000006,81.629999999999995,54.229999999999997,50.609999999999999,54.229999999999997,50.609999999999999,54.289999999999999,53.100000000000001,51.57,50.859999999999999,53.149999999999999,53.439999999999998,17.07,92.439999999999998,98.819999999999993,50.770000000000003,76.469999999999999,50.770000000000003,76.469999999999999,50.950000000000003,71.659999999999997,48.530000000000001,76.609999999999999,50.509999999999998,78.700000000000003
+CodeV-CL-7B,32.18,48.159999999999997,13.08,24.489999999999998,12.949999999999999,21.629999999999999,12.800000000000001,22.25,12.51,20.59,12.82,21.289999999999999,12.27,92.049999999999997,98.819999999999993,31.789999999999999,49.409999999999997,31.789999999999999,49.409999999999997,31.739999999999998,48.130000000000003,29.449999999999999,49.340000000000003,31.609999999999999,49.079999999999998
+CodeV-DS-6.7B,33.590000000000003,67.349999999999994,15,38.780000000000001,15,37.140000000000001,15.1,35.560000000000002,14.460000000000001,35.130000000000003,14.85,35.880000000000003,21.260000000000002,95.510000000000005,100,47.049999999999997,61.18,47.049999999999997,60,47.369999999999997,59.880000000000003,44.350000000000001,59.960000000000001,46.520000000000003,59.770000000000003

results/results_verilator.csv

@@ -0,0 +1,30 @@
+,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area,EM,Syntax (STX),Syntax (STX),Functionality (FNC),Functionality (FNC),Synthesis (SYN),Synthesis (SYN),Power,Power,Performance,Performance,Area,Area
+,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,VerilogEval S2R,RTLLM,RTL-Repo,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen,VerilogEval MC,VeriGen
+DeepSeek R1-0528,96.540000000000006,89.790000000000006,79.099999999999994,67.659999999999997,78.969999999999999,64.260000000000005,78.439999999999998,73.319999999999993,76.290000000000006,65.379999999999995,78.200000000000003,70.859999999999999,-1,94.739999999999995,98.819999999999993,80,80,79.870000000000005,74.120000000000005,79.659999999999997,63.859999999999999,77.280000000000001,74.049999999999997,79.409999999999997,77.010000000000005
+DeepSeek R1,97.689999999999998,93.189999999999998,79.359999999999999,67.659999999999997,79.099999999999994,62.979999999999997,79.010000000000005,71.379999999999995,76.489999999999995,63.740000000000002,78.629999999999995,69.819999999999993,-1,97.439999999999998,95.290000000000006,77.819999999999993,74.120000000000005,77.689999999999998,74.120000000000005,77.299999999999997,64.579999999999998,75.030000000000001,74.420000000000002,76.939999999999998,77.150000000000006
+Llama 3.1 405B,88.209999999999994,83.400000000000006,56.789999999999999,50.210000000000001,56.409999999999997,43.829999999999998,55.159999999999997,53.869999999999997,54.009999999999998,33.829999999999998,53.990000000000002,45.729999999999997,34.619999999999997,90.510000000000005,95.290000000000006,57.689999999999998,60,56.670000000000002,55.289999999999999,55.950000000000003,51.57,54.090000000000003,54.869999999999997,55.259999999999998,57.670000000000002
+Qwen3 236B A22B,93.969999999999999,78.719999999999999,75.769999999999996,56.170000000000002,75.379999999999995,50.210000000000001,75,57.009999999999998,72.939999999999998,47.850000000000001,74.780000000000001,52.450000000000003,41.939999999999998,83.719999999999999,88.239999999999995,68.969999999999999,62.350000000000001,68.459999999999994,62.350000000000001,68.299999999999997,54.630000000000003,65.989999999999995,62.630000000000003,68.349999999999994,62.780000000000001
+Qwen2.5 72B,83.849999999999994,82.980000000000004,55,48.509999999999998,54.490000000000002,45.960000000000001,53.969999999999999,48.450000000000003,51.299999999999997,47.340000000000003,52.840000000000003,49.68,37.439999999999998,83.079999999999998,87.060000000000002,54.740000000000002,50.590000000000003,54.359999999999999,50.590000000000003,53.359999999999999,50.049999999999997,51.200000000000003,50.560000000000002,52.759999999999998,52.229999999999997
+Llama 3.(1-3) 70B,68.329999999999998,79.150000000000006,40.899999999999999,45.960000000000001,40.899999999999999,40.43,40.770000000000003,42.119999999999997,38.329999999999998,39.340000000000003,40.420000000000002,40.920000000000002,28.719999999999999,86.030000000000001,87.060000000000002,43.719999999999999,63.530000000000001,43.590000000000003,63.530000000000001,43.549999999999997,62.289999999999999,41.57,63.520000000000003,42.939999999999998,63.539999999999999
+QwQ 32B,89.230000000000004,85.959999999999994,67.560000000000002,58.299999999999997,67.180000000000007,53.619999999999997,67.040000000000006,57.289999999999999,64.780000000000001,53.340000000000003,66.439999999999998,57.560000000000002,-1,59.619999999999997,87.060000000000002,40.130000000000003,65.879999999999995,39.490000000000002,64.709999999999994,39.259999999999998,60.950000000000003,37.490000000000002,64.480000000000004,39.170000000000002,66.310000000000002
+Qwen2.5 32B,90.900000000000006,87.659999999999997,55.899999999999999,50.210000000000001,55.259999999999998,46.810000000000002,55.560000000000002,50.659999999999997,53.100000000000001,47.409999999999997,54.020000000000003,51.369999999999997,28.93,95.510000000000005,85.879999999999995,45,49.409999999999997,43.719999999999999,49.409999999999997,43.719999999999999,46.409999999999997,42.490000000000002,49.890000000000001,42.950000000000003,49.609999999999999
+StarChat2 15B v0.1,88.719999999999999,89.359999999999999,38.850000000000001,48.509999999999998,38.850000000000001,45.960000000000001,38.920000000000002,48.950000000000003,36.18,45.049999999999997,38.810000000000002,48.630000000000003,13.42,81.409999999999997,91.760000000000005,36.539999999999999,63.530000000000001,36.280000000000001,63.530000000000001,36.189999999999998,57.509999999999998,35.109999999999999,63.229999999999997,36.240000000000002,65
+DeepSeek R1 Distill Qwen 14B,42.82,36.600000000000001,25.260000000000002,20.43,25,17.02,24.809999999999999,18.620000000000001,23.68,17.18,24.829999999999998,18.07,-1,45.640000000000001,44.710000000000001,25.379999999999999,29.41,25.260000000000002,29.41,24.98,25.920000000000002,23.609999999999999,29.43,24.690000000000001,29.649999999999999
+CodeLlama 70B,67.310000000000002,77.450000000000003,34.740000000000002,38.719999999999999,34.490000000000002,36.170000000000002,34.030000000000001,39.18,32.880000000000003,36.140000000000001,33.829999999999998,37.93,24.329999999999998,92.560000000000002,88.239999999999995,35.770000000000003,52.939999999999998,35.380000000000003,51.759999999999998,34.810000000000002,50.619999999999997,32.340000000000003,51.890000000000001,34.520000000000003,52.359999999999999
+DeepSeek Coder 33B,64.489999999999995,88.939999999999998,23.59,46.380000000000003,23.59,44.259999999999998,23.370000000000001,44.539999999999999,23.32,41.539999999999999,22.739999999999998,45.039999999999999,24.579999999999998,76.540000000000006,88.239999999999995,40.130000000000003,50.590000000000003,39.869999999999997,50.590000000000003,38.700000000000003,51.5,37.329999999999998,50.219999999999999,38.399999999999999,51.630000000000003
+QwenCoder 2.5 32B,90.260000000000005,81.280000000000001,46.539999999999999,47.659999999999997,46.409999999999997,45.109999999999999,45.950000000000003,48.810000000000002,44.420000000000002,45.039999999999999,45.57,47.359999999999999,31.07,86.150000000000006,85.879999999999995,48.210000000000001,54.119999999999997,47.560000000000002,54.119999999999997,47.350000000000001,54.590000000000003,45.350000000000001,54.119999999999997,46.780000000000001,55.140000000000001
+DeepCoder 14B,45.130000000000003,42.549999999999997,28.719999999999999,25.530000000000001,28.719999999999999,22.98,28.579999999999998,26.059999999999999,26.859999999999999,22.940000000000001,28.16,24.140000000000001,-1,63.460000000000001,49.409999999999997,34.359999999999999,37.649999999999999,34.100000000000001,35.289999999999999,34.170000000000002,32.789999999999999,32.579999999999998,35.289999999999999,33.409999999999997,35.280000000000001
+QwenCoder 2.5 14B,80.900000000000006,82.980000000000004,39.869999999999997,50.640000000000001,38.969999999999999,47.659999999999997,37.490000000000002,47.82,36.369999999999997,46.979999999999997,36.729999999999997,48.399999999999999,37.530000000000001,81.409999999999997,84.709999999999994,43.079999999999998,50.590000000000003,42.439999999999998,50.590000000000003,42.020000000000003,49.509999999999998,40.479999999999997,50.859999999999999,42.109999999999999,50.149999999999999
+SeedCoder 8B,96.409999999999997,89.359999999999999,54.359999999999999,51.490000000000002,54.229999999999997,48.509999999999998,53.68,51.520000000000003,51.5,47.539999999999999,52.560000000000002,51.700000000000003,28.23,78.209999999999994,91.760000000000005,38.210000000000001,47.060000000000002,38.079999999999998,43.530000000000001,38.100000000000001,37.890000000000001,36.090000000000003,43.770000000000003,37.539999999999999,43.869999999999997
+SeedCoder 8B Reasoning,68.719999999999999,56.170000000000002,48.079999999999998,34.039999999999999,48.079999999999998,30.640000000000001,48.270000000000003,33.409999999999997,45.780000000000001,29.859999999999999,47,31.27,-1,84.489999999999995,80,48.969999999999999,63.530000000000001,48.850000000000001,63.530000000000001,48.68,54.979999999999997,46.32,62.969999999999999,47.530000000000001,66.099999999999994
+OpenCoder 8B,80,80,27.949999999999999,43.829999999999998,27.949999999999999,42.130000000000003,27.52,42.439999999999998,26.07,40.979999999999997,27.280000000000001,42.460000000000001,16.170000000000002,81.030000000000001,94.120000000000005,36.789999999999999,49.409999999999997,36.149999999999999,44.710000000000001,35.890000000000001,41.219999999999999,34.240000000000002,44.579999999999998,35.899999999999999,45.079999999999998
+QwenCoder 2.5 7B,19.23,83.400000000000006,7.1799999999999997,40.850000000000001,7.1799999999999997,38.719999999999999,7.0199999999999996,42.310000000000002,7.1500000000000004,38.710000000000001,7.0800000000000001,41.200000000000003,28.329999999999998,76.150000000000006,90.590000000000003,35,48.240000000000002,34.740000000000002,48.240000000000002,34.619999999999997,44.93,32.799999999999997,48.399999999999999,34.649999999999999,49.759999999999998
+DeepSeek Coder 6.7B,86.030000000000001,89.359999999999999,31.280000000000001,45.960000000000001,31.149999999999999,42.979999999999997,31.16,47.369999999999997,29.149999999999999,43.299999999999997,31.079999999999998,48.170000000000002,24.629999999999999,69.099999999999994,83.530000000000001,31.670000000000002,37.649999999999999,30.129999999999999,37.649999999999999,30.190000000000001,36.450000000000003,28.16,37.469999999999999,29.82,37.659999999999997
+CodeV R1 Distill Qwen 7B,56.409999999999997,79.150000000000006,33.969999999999999,50.640000000000001,33.850000000000001,48.939999999999998,33.799999999999997,50.920000000000002,32.490000000000002,49.039999999999999,33.710000000000001,50.670000000000002,-1,94.230000000000004,87.060000000000002,22.18,63.530000000000001,22.18,63.530000000000001,22.210000000000001,57.75,20.73,63.240000000000002,22.079999999999998,63.890000000000001
+HaVen-CodeQwen,93.969999999999999,84.260000000000005,46.789999999999999,47.229999999999997,46.280000000000001,44.259999999999998,45.469999999999999,42.270000000000003,44.289999999999999,42.07,45.5,44.270000000000003,25.140000000000001,95,95.290000000000006,50,67.060000000000002,50,67.060000000000002,48.509999999999998,65.700000000000003,47.560000000000002,67.609999999999999,48.140000000000001,66.799999999999997
+CodeV-QW-7B,45.259999999999998,71.489999999999995,20.640000000000001,36.170000000000002,20,29.359999999999999,19.940000000000001,30.309999999999999,19.739999999999998,24.52,19.879999999999999,28.48,20.940000000000001,92.689999999999998,96.469999999999999,51.539999999999999,57.649999999999999,51.409999999999997,57.649999999999999,51.520000000000003,56.439999999999998,48.710000000000001,57.560000000000002,51.310000000000002,58.350000000000001
+RTLCoder Mistral,56.030000000000001,35.32,25.379999999999999,17.02,25.379999999999999,16.170000000000002,25.52,15.859999999999999,23.18,15.34,25.539999999999999,15.84,14.77,62.310000000000002,87.060000000000002,27.309999999999999,56.469999999999999,27.18,56.469999999999999,26.93,52.350000000000001,25.43,56.549999999999997,26.949999999999999,59.170000000000002
+RTLCoder DeepSeek,84.739999999999995,76.170000000000002,40.509999999999998,42.130000000000003,40.380000000000003,37.450000000000003,39.729999999999997,36.75,38.439999999999998,35.289999999999999,39.399999999999999,36.390000000000001,19.350000000000001,79.230000000000004,85.879999999999995,36.409999999999997,55.289999999999999,36.409999999999997,55.289999999999999,36.490000000000002,50.789999999999999,34.509999999999998,55.219999999999999,36.210000000000001,54.689999999999998
+OriGen,96.790000000000006,85.109999999999999,54.359999999999999,53.189999999999998,53.850000000000001,51.060000000000002,54.079999999999998,54.170000000000002,51.07,51.310000000000002,53.009999999999998,54.390000000000001,17.07,92.560000000000002,96.469999999999999,50.509999999999998,74.120000000000005,50.509999999999998,74.120000000000005,50.829999999999998,69.299999999999997,48.280000000000001,74.260000000000005,50.409999999999997,76.349999999999994
+CodeV-CL-7B,32.82,54.469999999999999,13.460000000000001,29.359999999999999,13.33,22.550000000000001,13.19,23.190000000000001,13.26,21.469999999999999,13.19,22.190000000000001,12.27,92.180000000000007,98.819999999999993,31.789999999999999,49.409999999999997,31.789999999999999,49.409999999999997,31.739999999999998,48.130000000000003,29.829999999999998,49.340000000000003,31.609999999999999,49.079999999999998
+CodeV-DS-6.7B,34.359999999999999,69.790000000000006,14.74,41.280000000000001,14.359999999999999,37.869999999999997,14.380000000000001,35.890000000000001,14.130000000000001,35.740000000000002,14.19,36.18,21.260000000000002,95.769999999999996,100,46.670000000000002,61.18,46.670000000000002,61.18,47.07,61.060000000000002,44.189999999999998,61.130000000000003,46.469999999999999,60.950000000000003