scripts/dashboard.py

"""
Launch with:
    streamlit run scripts/dashboard.py

Relies on the directory structure produced by analysis.py:
outputs/grid/<dataset>/<config>/{aggregates.yaml, rq1.yaml, ...}
"""
from __future__ import annotations

import json
import yaml
from pathlib import Path

import pandas as pd
import streamlit as st
import matplotlib.pyplot as plt

BASE_DIR = Path("outputs/grid")
METRIC_KEY = "rag_score"

# --------------------------------------------------------------------- Sidebar
st.sidebar.title("RAG-Eval Dashboard")

if not BASE_DIR.exists():
    st.sidebar.error(f"Folder {BASE_DIR} not found – run experiments first.")
    st.stop()

datasets = sorted([p.name for p in BASE_DIR.iterdir() if p.is_dir()])
dataset = st.sidebar.selectbox("Dataset", datasets)
conf_dir = BASE_DIR / dataset
configs = sorted([p.name for p in conf_dir.iterdir() if p.is_dir()])
sel_cfgs = st.sidebar.multiselect("Configurations", configs, default=configs)

if not sel_cfgs:
    st.warning("Select at least one configuration.")
    st.stop()

# ---------------------------------------------------------------- Load helpers
def _yaml(path: Path): return yaml.safe_load(path.read_text())
def _jsonl(path: Path): return [json.loads(l) for l in path.read_text().splitlines()]

# ---------------------------------------------------------------- Main view
st.title(f"Dataset: {dataset}")

# ── Aggregated metrics table ────────────────────────────────────────────────
agg = {c: _yaml(conf_dir / c / "aggregates.yaml") for c in sel_cfgs}
agg_df = pd.DataFrame(agg).T
st.subheader("Aggregated metrics")
st.dataframe(agg_df, use_container_width=True)

# ── Bar chart of rag_score means ────────────────────────────────────────────
st.subheader(f"Mean {METRIC_KEY}")
fig, ax = plt.subplots()
agg_df[METRIC_KEY].plot.bar(ax=ax)
ax.set_ylabel(METRIC_KEY)
ax.set_ylim(0, 1)
st.pyplot(fig)

# ── Scatter MRR vs Correctness per config ───────────────────────────────────
st.subheader("MRR vs Human Correctness")
cols = st.columns(len(sel_cfgs))
for col, cfg in zip(cols, sel_cfgs):
    rows = _jsonl(conf_dir / cfg / "results.jsonl")
    x = [r["metrics"].get("mrr", float("nan")) for r in rows]
    y = [1 if r.get("human_correct") else 0 for r in rows]
    fig, ax = plt.subplots()
    ax.scatter(x, y, alpha=0.5)
    ax.set(title=cfg, xlabel="MRR", ylabel="Correct?")
    col.pyplot(fig)

# ── Pairwise Wilcoxon-Holm table (rag_score) ────────────────────────────────
wh_path = conf_dir / "wilcoxon_rag_holm.yaml"
if wh_path.exists():
    st.subheader("Pairwise Wilcoxon-Holm (rag_score)")
    wh_df = pd.Series(_yaml(wh_path), name="p_adj").to_frame()
    st.dataframe(wh_df)
else:
    st.info("Wilcoxon table not found – run_grid_experiments.py computes it.")

# ── Research-question YAMLs ─────────────────────────────────────────────────
rq_tabs = st.tabs([f"{cfg}" for cfg in sel_cfgs])
for tab, cfg in zip(rq_tabs, sel_cfgs):
    with tab:
        for rq in ("rq1", "rq2", "rq3", "rq4"):
            path = conf_dir / cfg / f"{rq}.yaml"
            if path.exists():
                st.markdown(f"**{rq.upper()}**")
                st.json(_yaml(path))
            else:
                st.markdown(f"*{rq.upper()} – not available*")

# ── Raw results download ────────────────────────────────────────────────────
st.sidebar.subheader("Download")
for cfg in sel_cfgs:
    st.sidebar.download_button(
        label=f"{cfg} results.jsonl",
        data=(conf_dir / cfg / "results.jsonl").read_bytes(),
        file_name=f"{dataset}_{cfg}_results.jsonl",
        mime="application/jsonl",
    )