improved result.json to include api call details, added a new prompt

.gitignore

@@ -0,0 +1,10 @@
+# Ignore all subdirectories in user_submit
+**/user_submit/*/
+**/__pycache__/
+**/*.pyc
+
+# But re-include these four
+!**/user_submit/llm/
+!**/user_submit/rag/
+!**/user_submit/mt/
+!**/user_submit/12-02-14:29:30/

.vscode/launch.json

@@ -0,0 +1,15 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal"
+        }
+    ]
+}

eval_models_list.txt

@@ -1,3 +1,5 @@
 gpt-4o-mini
 gpt-4o-2024-05-13
 gpt-4o-2024-08-06
+gpt-4.1
+gpt-4.1-mini

eval_prompt_list.txt

@@ -1,3 +1,4 @@
 llmbar
 translation_pair
 rag_pair_kr
+translation_new

streamlit_app_local/eval_models_list.txt

@@ -1,3 +0,0 @@
-gpt-4o-mini
-gpt-4o-2024-05-13
-gpt-4o-2024-08-06

streamlit_app_local/eval_models_list.txt

@@ -0,0 +1 @@
+../eval_models_list.txt

streamlit_app_local/eval_prompt_list.txt

@@ -1,5 +0,0 @@
-llmbar
-llmbar_brief
-translation_pair
-rag_pair_kr
-contextual (WIP)

streamlit_app_local/eval_prompt_list.txt

@@ -0,0 +1 @@
+../eval_prompt_list.txt

streamlit_app_local/pages/see_results.py

@@ -2,6 +2,7 @@ import pandas as pd
 import streamlit as st
 from varco_arena_core.prompts import load_prompt
 
+from pprint import pprint
 import analysis_utils as au
 from analysis_utils import number_breakdown_from_df
 from app import VA_ROOT
@@ -248,11 +249,12 @@ def main():
                         out_b="{out_b}",
                         task=task,
                     )
-                    if eval_prompt_name == "translation_pair":
+                    if eval_prompt_name in ["translation_pair", "translation_new"]:
                         kwargs["source_lang"] = "{source_lang}"
                         kwargs["target_lang"] = "{target_lang}"
                     prompt_cmpl = prompt.complete_prompt(**kwargs)
                     for msg in prompt_cmpl:
+                        pprint(msg)
                         st.markdown(f"**{msg['role']}**")
                         st.info(show_linebreak_in_md(escape_markdown(msg["content"])))
 

varco_arena/main.py

@@ -134,13 +134,6 @@ if __name__ == "__main__":
         "-p",
         "--evalprompt",
         default="llmbar_brief",
-        choices=[
-            "llmbar_brief",
-            "llmbar",
-            "translation_pair",
-            "rag_pair_kr",
-            # "contextual_pair",
-        ],
     )
 
     parser.add_argument(

varco_arena/varco_arena_core/eval_utils.py

@@ -138,6 +138,7 @@ async def async_query_openai(
 
     increase_match_count()  # you're hacky Jumin...
 
+    normalized_result["api_call_kwargs"] = kwargs
     return normalized_result, resp
 
 

varco_arena/varco_arena_core/league.py

@@ -59,6 +59,7 @@ class League:
                         "round": "league",
                         "match_order_in_round": "league",
                         "tstamp": now_time,
+                        "api_call_kwargs": match_result[0]["api_call_kwargs"],
                         # "logs": match.match_metainfo_log[0],
                     },
                 ]

varco_arena/varco_arena_core/match.py

@@ -6,7 +6,6 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
 from .eval_utils import async_eval_w_prompt
 
-
 class Match:
     def __init__(
         self,

varco_arena/varco_arena_core/prompts/__init__.py

@@ -8,12 +8,14 @@ from .llmbar import LLMBarPrompt
 from .llmbar_brief import LLMBarBriefPrompt
 from .rag_pair_kr import RagPairKRPrompt
 from .translation_pair import TranslationPairPrompt
+from .translation_new import TranslationNewPrompt
 
 NAME2PROMPT_CLS = dict(
     llmbar_brief=LLMBarBriefPrompt(),
     llmbar=LLMBarPrompt(),
     translation_pair=TranslationPairPrompt(),
     rag_pair_kr=RagPairKRPrompt(),
+    translation_new=TranslationNewPrompt(),
     # contextual_vqa = Contextual_VQA(),
     # contextual_ocr = Contextual_OCR(),
 )
@@ -24,8 +26,8 @@ def load_prompt(
         "llmbar_brief",
         "llmbar",
         "translation_pair",
+        "translation_new",
         "rag_pair_kr",
-        # "contextual_pair"
     ],
     task: str = "",  # used for further prompt variation (eval prompt might depend on task.)
 ):