Resolved tests issues

evaluation/config.py

@@ -38,7 +38,7 @@ class RetrieverConfig:
     index_path: Optional[Union[str, Path]] = None  # alias for bm25_index
 
     # Specific to BM25
-    bm25_index: Optional[Union[str, Path]] = None
+    bm25_idx: Optional[Union[str, Path]] = None
     doc_store: Optional[Union[str, Path]] = None
 
     # For dense-only
@@ -53,7 +53,7 @@ class RetrieverConfig:
     def __post_init__(self):
         # If index_path is provided (legacy), use it as bm25_index
         if self.index_path:
-            self.bm25_index = self.index_path
+            self.bm25_idx = self.index_path
 
 
 @dataclass

evaluation/metrics/composite.py

@@ -5,12 +5,12 @@ from typing import Mapping
 import math
 
 
-def harmonic_mean(scores: Mapping[str, float], eps: float = 1e-6) -> float:
+def harmonic_mean(scores: Mapping[str, float]) -> float:
     """Compute the harmonic mean of positive scores."""
     if not scores:
         return 0.0
-    inv_sum = sum(1.0 / (v + eps) for v in scores.values() if v > 0)
-    return len(scores) / inv_sum if inv_sum else 0.0
+    inv_sum = sum(1.0 / (v) for v in scores.values() if v > 0)
+    return len(scores) / inv_sum if inv_sum and inv_sum != 0 else 0.0
 
 
 def rag_score(scores: Mapping[str, float]) -> float:

evaluation/pipeline.py

@@ -22,29 +22,18 @@ class RAGPipeline:
         self.generator = HFGenerator(
             model_name=cfg.generator.model_name, device=cfg.generator.device
         )
-        self.reranker = (
-            CrossEncoderReranker(
+        if cfg.reranker.enable:
+            self.reranker = CrossEncoderReranker(
                 cfg.reranker.model_name,
                 device=cfg.reranker.device,
-                max_len=cfg.reranker.max_length,
+                max_length=cfg.reranker.max_length,
             )
-            if cfg.reranker.enable
-            else None
-        )
+        else:
+            self.reranker = None
 
     # ---------------------------------------------------------------------
     # Public API
     # ---------------------------------------------------------------------
-    def run_queries(self, queries: list[dict[str, Any]]) -> list[dict[str, Any]]:
-        """Accepts a list of {'question': str, 'id': Any}, returns list of result dicts."""
-        results: list[dict[str, Any]] = []
-        for entry in queries:
-            q = entry.get("question", "")
-            doc_id = entry.get("id")
-            answer = self.run_query(q)
-            results.append({"id": doc_id, "question": q, "answer": answer})
-        return results
-
     def run(self, question: str) -> Dict[str, Any]:
         """Retrieve context and generate answer."""
         logger.info("Question: %s", question)
@@ -58,22 +47,40 @@ class RAGPipeline:
 
     __call__ = run  # alias
 
+    def run_queries(self, queries: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        """Accepts a list of {'question': str, 'id': Any}, returns list of result dicts."""
+        results: list[dict[str, Any]] = []
+        for entry in queries:
+            q = entry.get("question", "")
+            doc_id = entry.get("id")
+            answer = self.run(q)
+            results.append({"id": doc_id, "question": q, "answer": answer})
+        return results
     # ---------------------------------------------------------------------
     # Private helpers
     # ---------------------------------------------------------------------
     def _build_retriever(self, cfg: PipelineConfig) -> Retriever:
-        name = cfg.retriever.name
         r=cfg.retriever
+        name = r.name
         if name == "bm25":
-             return bm25.BM25Retriever(
-                index_path=str(r.bm25_index), doc_store_path=str(r.doc_store))
+            return bm25.BM25Retriever(bm25_idx=str(r.bm25_index), doc_store_path=str(r.doc_store))
         if name == "dense":
-            return dense.DenseRetriever(faiss_index=str(r.faiss_index),doc_store=r.doc_store,model_name=r.model_name,embedder_cache=r.embedder_cache,device=r.device)
+            return dense.DenseRetriever(
+                faiss_index=str(r.faiss_index),
+                doc_store=str(r.doc_store),
+                model_name=r.model_name,
+                embedder_cache=str(r.embedder_cache) if r.embedder_cache else None,
+                device=r.device,
+            )
         if name == "hybrid":
             return hybrid.HybridRetriever(
-                bm25_idx=str(r.bm25_index),
-                dense_idx=str(r.faiss_index),
+                str(r.bm25_index),
+                str(r.faiss_index),
+                doc_store=str(r.doc_store),
                 alpha=r.alpha,
+                model_name=r.model_name,
+                embedder_cache=str(r.embedder_cache) if r.embedder_cache else None,
+                device=r.device,
             )
         raise ValueError(f"Unsupported retriever '{name}'")
 

evaluation/retrievers/bm25.py

@@ -18,11 +18,11 @@ class BM25Retriever(Retriever):
 
     def __init__(
         self,
-        index_path: str | None,
+        bm25_idx: str | None,
         doc_store_path: str | None = None,
         threads: int = 1,
     ):
-        if index_path is None:
+        if bm25_idx is None:
             raise ValueError("BM25 retriever requires a path to a Pyserini index.")
 
         # ❶ Attempt to import SimpleSearcher. If it fails (ImportError or Java mismatch),
@@ -39,24 +39,24 @@ class BM25Retriever(Retriever):
             )
             SimpleSearcher = None
 
-        self.index_path = index_path
+        self.bm25_idx = bm25_idx
         self.doc_store_path = doc_store_path
         self.threads = threads
         self.searcher = None
 
         # ❷ If the index folder does not exist, attempt to build it from doc_store_path
-        if not Path(index_path).exists():
+        if not Path(bm25_idx).exists():
             if doc_store_path is None:
                 logger.warning(
                     "BM25 index %s not found and no `doc_store_path` supplied. "
                     "BM25Retriever.retrieve() will return no hits.",
-                    index_path,
+                    bm25_idx,
                 )
             else:
                 try:
                     logger.info(
                         "BM25 index %s missing – building from %s ...",
-                        index_path,
+                        bm25_idx,
                         doc_store_path,
                     )
                     self._build_index(Path(doc_store_path), index_path, threads)
@@ -78,7 +78,7 @@ class BM25Retriever(Retriever):
                 logger.warning(
                     "Failed to instantiate SimpleSearcher on %s (%s). "
                     "BM25Retriever.retrieve() will return no hits.",
-                    index_path,
+                    bm25_idx,
                     e,
                 )
                 self.searcher = None

evaluation/retrievers/hybrid.py

@@ -1,11 +1,8 @@
-"""Hybrid retriever that combines sparse and dense scores (linear sum)."""
-
 from __future__ import annotations
-from typing import List, Optional
-from pathlib import Path
 import logging
+from typing import List, Optional
 
-from .base import Retriever, Context
+from .base import Context, Retriever
 from .bm25 import BM25Retriever
 from .dense import DenseRetriever
 
@@ -13,32 +10,63 @@ logger = logging.getLogger(__name__)
 
 
 class HybridRetriever(Retriever):
-    """Combine BM25 and Dense retrievers by score normalisation and sum."""
+    """Combine BM25 and Dense retrievers by normalising and summing scores."""
 
-    def __init__(self, bm25_idx: str, faiss_idx: str, *, doc_store: str, alpha: float = 0.5, model_name: str = "sentence-transformers/all-MiniLM-L6-v2", embedder_cache: Optional[str] = None, device: str = "cpu"):
-        self.bm25 = BM25Retriever(index_path=bm25_idx, doc_store_path=doc_store)
+    def __init__(
+        self,
+        bm25_idx: str,
+        faiss_idx: str,
+        doc_store: str,
+        *,
+        alpha: float = 0.5,
+        model_name: str = "sentence-transformers/all-MiniLM-L6-v2",
+        embedder_cache: Optional[str] = None,
+        device: str = "cpu",
+    ):
+        # 1) BM25 retriever
+        self.bm25 = BM25Retriever(bm25_idx, doc_store_path=doc_store)
+
+        # 2) Dense retriever
         self.dense = DenseRetriever(
             faiss_index=faiss_idx,
             doc_store=doc_store,
             model_name=model_name,
             embedder_cache=embedder_cache,
-            device=device)
+            device=device,
+        )
+
         if not 0 <= alpha <= 1:
-            raise ValueError("alpha must be in [0, 1]")
+            raise ValueError("alpha must be in [0, 1]")
         self.alpha = alpha
 
     def retrieve(self, query: str, *, top_k: int = 5) -> List[Context]:
-        sparse_ctxs = {c.id: c for c in self.sparse.retrieve(query, top_k=top_k)}
-        dense_ctxs = {c.id: c for c in self.dense.retrieve(query, top_k=top_k)}
+        # 1) Get sparse hits
+        sparse_hits = self.bm25.retrieve(query, top_k=top_k)
+        sparse_dict = {ctx.id: ctx for ctx in sparse_hits}
 
-        ids = list(set(sparse_ctxs) | set(dense_ctxs))
+        # 2) Get dense hits
+        dense_hits = self.dense.retrieve(query, top_k=top_k)
+        dense_dict = {ctx.id: ctx for ctx in dense_hits}
+
+        # 3) Union of all IDs
+        all_ids = set(sparse_dict) | set(dense_dict)
         merged: List[Context] = []
-        for doc_id in ids:
-            sparse_score = sparse_ctxs.get(doc_id, Context(doc_id, "", 0.0)).score
-            dense_score = dense_ctxs.get(doc_id, Context(doc_id, "", 0.0)).score
-            score = self.alpha * sparse_score + (1 - self.alpha) * dense_score
-            text = sparse_ctxs.get(doc_id, dense_ctxs.get(doc_id)).text  # type: ignore
-            merged.append(Context(id=doc_id, text=text, score=score))
 
+        for doc_id in all_ids:
+            s_score = sparse_dict.get(doc_id, Context(doc_id, "", 0.0)).score
+            d_score = dense_dict.get(doc_id, Context(doc_id, "", 0.0)).score
+
+            combined_score = self.alpha * s_score + (1 - self.alpha) * d_score
+
+            # Prefer the text from whichever retriever has this doc_id present;
+            # if only one side has it, grab that text.
+            if doc_id in sparse_dict:
+                text = sparse_dict[doc_id].text
+            else:
+                text = dense_dict[doc_id].text
+
+            merged.append(Context(id=doc_id, text=text, score=combined_score))
+
+        # 4) Sort by score descending
         merged.sort(key=lambda c: c.score, reverse=True)
         return merged[:top_k]

evaluation/stats/robustness.py

@@ -79,4 +79,5 @@ def chi2_error_propagation(
         chi2, p, dof, expected = chi2_contingency(table)
         return dict(chi2=chi2, p=p, dof=dof, expected=expected, table=table)
     except ValueError:
-        return dict(chi2=0.0, p=1.0, dof=dof, expected=expected, table=table)
+        default_expected = [[0, 0], [0, 0]]
+        return dict(chi2=0.0, p=1.0, dof=0, expected=default_expected, table=table)

evaluation/stats/significance.py

@@ -41,4 +41,7 @@ def holm_bonferroni(pvalues: Mapping[str, float]) -> Mapping[str, float]:
 
 def delta_metric(base: Sequence[float], new: Sequence[float]) -> list[float]:
     """Compute per‐element differences `new[i] - base[i]` as a list of floats."""
-    return [float(n - b) for b, n in zip(base, new)]
+    diffs: list[float] = []
+    for b, n in zip(base, new):
+        diffs.append(float(n - b))
+    return diffs

tests/test_metrics.py

@@ -25,7 +25,6 @@ def test_retrieval_metrics_simple():
     assert recall_at_k(retrieved, relevant, 2) == pytest.approx(1 / 3, rel=1e-6)
     assert recall_at_k(retrieved, relevant, 4) == pytest.approx(2 / 3, rel=1e-6)
     assert mean_reciprocal_rank(retrieved, relevant) == pytest.approx(0.5, rel=1e-6)
-    # AP = (1/2 + 2/4)/3 = 1/3
     assert average_precision(retrieved, relevant) == pytest.approx(1 / 3, rel=1e-6)