Implemened L3Score from SPIQA datase paper

L3Score.py

@@ -0,0 +1,255 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+L3Score metric to score the quality of a free-form answer given a question and a ground-truth answer.
+The metric is based on the log-probability of the Yes/No token of an LLM judge.
+Metric is based on the paper: https://arxiv.org/pdf/2407.09413
+"""
+
+import os
+
+import evaluate
+import datasets
+import numpy as np
+
+from langchain.chat_models.base import init_chat_model
+
+
+_CITATION = """\
+@article{pramanick2024spiqa,
+  title={Spiqa: A dataset for multimodal question answering on scientific papers},
+  author={Pramanick, Shraman and Chellappa, Rama and Venugopalan, Subhashini},
+  journal={arXiv preprint arXiv:2407.09413},
+  year={2024}
+}
+"""
+
+_DESCRIPTION = """\
+Implements the L3Score metric to score the quality of a free-form answer given a question and a ground-truth answer.
+The metric is based on the log-probability of the Yes/No token of an LLM judge.
+Metric is based on the paper: https://arxiv.org/pdf/2407.09413
+"""
+
+
+_KWARGS_DESCRIPTION = """
+Implements the L3Score metric to score the quality of a free-form answer given a question and a ground-truth answer.
+Args:
+    questions: list of questions to score. Each question should be a string.
+    predictions: list of predictions to score. Each predictions
+        should be a string.
+    references: list of reference for each prediction. Each
+        reference should be a string.
+Returns:
+    L3Score: mean L3Score for all (question, prediction, reference) triplets.
+Examples:
+    Example 1: High certainty the prediction is the same as the ground-truth.
+    >>> L3Score = evaluate.load("L3Score")
+    >>> L3Score.compute(questions=["What is the capital of France?"], predictions=["Paris"], references=["Paris"], api_key="your-openai-api-key", provider="openai", model="gpt-4o-mini")
+    {'L3Score': 0.99...}
+
+    Example 2: High certainty the prediction is not the same as the ground-truth.
+    >>> L3Score = evaluate.load("L3Score")  
+    >>> L3Score.compute(questions=["What is the capital of Germany?"], predictions=["Moscow"], references=["Berlin"], api_key="your-openai-api-key", provider="openai", model="gpt-4o-mini")
+    {'L3Score': 0.00...}
+"""
+
+
+PROVIDER_WITH_TOP_LOGPROBS = ["openai", "deepseek", "xai"]
+
+_PROMPT = "You are given a question, ground-truth answer, and a candidate answer. Question: {question} \nGround-truth answer: {gt} \nCandidate answer: {answer} \n\
+Is the semantic meaning of the ground-truth and candidate answers similar? Answer in one word - Yes or No."
+
+_SUFFIXES_TO_SCORE = [" yes", " yeah"]
+_COMPLEMENT_SUFFIXES = [" no"]
+
+NEGATIVE_INF = -1000.0
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class L3Score(evaluate.Metric):
+    """
+    L3Score metric to score the quality of a free-form answer given a question and a ground-truth answer.
+    The metric is based on the log-probability of the Yes/No token of an LLM judge.
+    Metric is from the paper: https://arxiv.org/pdf/2407.09413
+    """
+
+    def _info(self):
+        return evaluate.MetricInfo(
+            module_type="metric",
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=datasets.Features(
+                {
+                    "questions": datasets.Value("string"),
+                    "predictions": datasets.Value("string"),
+                    "references": datasets.Value("string"),
+                }
+            ),
+            homepage="https://github.com/google/spiqa",
+            codebase_urls=[
+                "https://github.com/google/spiqa/blob/main/metrics/llmlogscore/llmlogscore.py"
+            ],
+            reference_urls=["https://arxiv.org/pdf/2407.09413","https://github.com/google/spiqa","https://huggingface.co/datasets/google/spiqa"],
+        )
+
+    def _download_and_prepare(self, dl_manager):
+        """Optional: download external resources useful to compute the scores"""
+        pass
+
+    def _verify_input(self, provider, model, api_key):
+        """Verify the input parameters"""
+
+        print(provider)
+        if provider not in PROVIDER_WITH_TOP_LOGPROBS:
+            raise ValueError(
+                "Provider must offer top_logprobs to use this metric, pick from {}".format(
+                    PROVIDER_WITH_TOP_LOGPROBS
+                )
+            )
+
+        if api_key == "":
+            raise ValueError("api_key is required")
+
+    def _get_llm(self, model, api_key):
+        """Get the LLM"""
+        llm = init_chat_model(model=model, api_key=api_key)
+        llm = llm.bind(logprobs=True, top_logprobs=5)
+        return llm
+
+    def _compute(
+        self,
+        questions,
+        predictions,
+        references,
+        api_key="",
+        provider="openai",
+        model="gpt-4o-mini",
+    ):
+        """Returns the scores"""
+
+        # Check whether llm can be initialized
+        self._verify_input(provider, model, api_key)
+
+        # Initialize the LLM
+        llm = self._get_llm(model, api_key)
+
+        L3Score = 0
+        count = 0
+        for question, prediction, reference in zip(questions, predictions, references):
+            response = llm.invoke(
+                (
+                    "human",
+                    _PROMPT.format(question=question, gt=reference, answer=prediction),
+                )
+            )
+            score = self._calculate_L3Score(
+                response.response_metadata["logprobs"]["content"][0]["top_logprobs"]
+            )
+            L3Score += score.item()
+            count += 1
+
+        if count > 0:
+            L3Score = L3Score / count
+
+        return {
+            "L3Score": L3Score,
+        }
+
+    def _calculate_L3Score(self, top_logprobs):
+        """
+        Calculates the L3 score for a given response.
+        """
+
+        normalized_suffixes = [self._normalize(suffix) for suffix in _SUFFIXES_TO_SCORE]
+        normalized_complement_suffixes = [
+            self._normalize(complement_suffix)
+            for complement_suffix in _COMPLEMENT_SUFFIXES
+        ]
+
+        suffix_logprob = NEGATIVE_INF
+        complement_logprob = NEGATIVE_INF
+        suffix_index = -1
+        complement_suffix_index = -1
+
+        for i, token_logprob in enumerate(top_logprobs):
+            if self._normalize(token_logprob["token"]) in normalized_suffixes:
+                suffix_logprob = token_logprob["logprob"]
+                suffix_index = i
+                break
+
+        for i, token_logprob in enumerate(top_logprobs):
+            if (
+                self._normalize(token_logprob["token"])
+                in normalized_complement_suffixes
+            ):
+                complement_suffix_index = i
+                complement_logprob = token_logprob["logprob"]
+                break
+
+        if suffix_index == -1 and complement_suffix_index == -1:
+            return 0.0
+
+        if suffix_index != -1 and complement_suffix_index != -1:
+            return self._renormalize_score(
+                yes_score=suffix_logprob, no_score=complement_logprob
+            )
+
+        lowest_logprob = top_logprobs[-1]["logprob"]
+        lowest_token_prob = np.exp(lowest_logprob)
+        sum_probs = sum(
+            [np.exp(token_logprob["logprob"]) for token_logprob in top_logprobs]
+        )
+        remaining_prob = 1 - sum_probs
+        min_prob = min(lowest_token_prob, remaining_prob)
+        if min_prob < 1e-8:
+            min_prob = 1e-8
+        reciprocal_logprob = np.log(min_prob)
+
+        if suffix_index != -1:
+            exclude_score = suffix_logprob
+            include_score = reciprocal_logprob
+        elif complement_suffix_index != -1:
+            exclude_score = reciprocal_logprob
+            include_score = complement_logprob
+
+        return self._renormalize_score(yes_score=exclude_score, no_score=include_score)
+
+    def _renormalize_score(self, yes_score: float, no_score: float) -> float:
+        """Renormalize the scores to be between 0 and 1."""
+        return 1 / (1 + np.exp(-(yes_score - no_score)))
+
+    def _normalize(self, text: str) -> str:
+        """Remove white space and lower case for normalized comparisons."""
+        return text.strip().lower()
+
+
+if __name__ == "__main__":
+
+    questions = ["What is the capital of France?", "What is the capital of Germany?"]
+    predictions = ["Paris", "Moscow"]
+    references = ["Paris", "Berlin"]
+
+    L3Score_test = L3Score()
+
+    results = L3Score_test.compute(
+        questions=questions,
+        predictions=predictions,
+        references=references,
+        api_key=os.environ["OPENAI_API_KEY"],
+        provider="deepseek",
+        model="deepseek-coder",
+    )
+    print(results)

README.md

@@ -1,50 +1,167 @@
 ---
-title: l3score
+title: L3Score
 datasets:
--  
+  - google/spiqa
 tags:
-- evaluate
-- metric
-description: "TODO: add a description here"
+  - evaluate
+  - metric
+  - semantic-similarity
+  - qa
+  - llm-eval
+description: >
+  L3Score is a metric for evaluating the semantic similarity of free-form answers in question answering tasks.
+  It uses log-probabilities of "Yes"/"No" tokens from a language model acting as a judge.
+  Based on the SPIQA benchmark: https://arxiv.org/pdf/2407.09413
 sdk: gradio
 sdk_version: 3.19.1
 app_file: app.py
 pinned: false
 ---
 
-# Metric Card for l3score
+# 🦢 Metric Card: L3Score
 
-***Module Card Instructions:*** *Fill out the following subsections. Feel free to take a look at existing metric cards if you'd like examples.*
+## 📌 Description
 
-## Metric Description
-*Give a brief overview of this metric, including what task(s) it is usually used for, if any.*
+**L3Score** evaluates how semantically close a model-generated answer is to a reference answer for a given question. It prompts a **language model as a judge** using the following format:
 
-## How to Use
-*Give general statement of how to use the metric*
+```text
+You are given a question, ground-truth answer, and a candidate answer.
 
-*Provide simplest possible example for using the metric*
+Question: {question}  
+Ground-truth answer: {gt}  
+Candidate answer: {answer}
 
-### Inputs
-*List all input arguments in the format below*
-- **input_field** *(type): Definition of input, with explanation if necessary. State any default value(s).*
+Is the semantic meaning of the ground-truth and candidate answers similar?  
+Answer in one word - Yes or No.
+```
 
-### Output Values
+The model's **log-probabilities** for "Yes" and "No" tokens are used to compute the score.
 
-*Explain what this metric outputs and provide an example of what the metric output looks like. Modules should return a dictionary with one or multiple key-value pairs, e.g. {"bleu" : 6.02}*
+### 🧮  Scoring Logic
 
-*State the range of possible values that the metric's output can take, as well as what in that range is considered good. For example: "This metric can take on any value between 0 and 100, inclusive. Higher scores are better."*
+Let $ l_{\text{yes}}$ and $ l_{\text{no}}$ be the log-probabilities of "Yes" and "No", respectively.
 
-#### Values from Popular Papers
-*Give examples, preferrably with links to leaderboards or publications, to papers that have reported this metric, along with the values they have reported.*
+If neither token is in the top-5:
 
-### Examples
-*Give code examples of the metric being used. Try to include examples that clear up any potential ambiguity left from the metric description above. If possible, provide a range of examples that show both typical and atypical results, as well as examples where a variety of input parameters are passed.*
+$$
+\text{L3Score} = 0
+$$
+
+If both are present:
+
+$$
+\text{L3Score} = \frac{\exp(l_{\text{yes}})}{\exp(l_{\text{yes}}) + \exp(l_{\text{no}})}
+$$
+
+If only one is present, the missing token’s probability is estimated using the minimum of the remaining mass or the least likely token in top-5.  
+See [SPIQA paper](https://arxiv.org/pdf/2407.09413) for details.
+
+---
+
+## 🚀 How to Use
+
+```python
+import evaluate
+
+l3score = evaluate.load("your-username/L3Score")
+
+questions = ["What is the capital of France?", "What is the capital of Germany?"]
+predictions = ["Paris", "Moscow"]
+references = ["Paris", "Berlin"]
+
+score = l3score.compute(
+    questions=questions,
+    predictions=predictions,
+    references=references,
+    api_key="your-openai-api-key",
+    provider="openai",
+    model="gpt-4o-mini"
+)
+
+print(score)
+# {'L3Score': 0.49...}
+```
+
+---
+
+### 🔠 Inputs
+
+| Name         | Type         | Description                                                                 |
+|--------------|--------------|-----------------------------------------------------------------------------|
+| `questions`  | `list[str]`  | The list of input questions.                                                |
+| `predictions`| `list[str]`  | Generated answers by the model being evaluated.                            |
+| `references` | `list[str]`  | Ground-truth or reference answers.                                         |
+| `api_key`    | `str`        | API key for the selected LLM provider.                                     |
+| `provider`   | `str`        | Must support top-n token log-probabilities (currently available: `"openai"`, `"deepseek","xai"`).|
+| `model`      | `str`        | Name of the evaluation LLM (e.g., `"gpt-4o-mini"`).                         |
+
+---
+
+### 📄 Output
+
+A dictionary with a single key:
+
+```python
+{"L3Score": float}
+```
+
+The value is the **average score** over all (question, prediction, reference) triplets.
+
+---
+
+## 💡 Examples
+
+```python
+l3score = evaluate.load("your-username/L3Score")
+
+score = l3score.compute(
+    questions=["What is the capital of France?"],
+    predictions=["Paris"],
+    references=["Paris"],
+    api_key="your-openai-api-key",
+    provider="openai",
+    model="gpt-4o-mini"
+)
+# {'L3Score': 0.99...}
+
+score = l3score.compute(
+    questions=["What is the capital of Germany?"],
+    predictions=["Moscow"],
+    references=["Berlin"],
+    api_key="your-openai-api-key",
+    provider="openai",
+    model="gpt-4o-mini"
+)
+# {'L3Score': 0.00...}
+```
+
+---
+
+## ⚠️ Limitations and Bias
+
+- Requires models that expose **top-n token log-probabilities** (e.g., OpenAI, DeepSeek, Groq).
+- Scores are **only comparable when using the same judge model**.
+
+---
+
+## 📖 Citation
+
+```bibtex
+@article{pramanick2024spiqa,
+  title={SPIQA: A Dataset for Multimodal Question Answering on Scientific Papers},
+  author={Pramanick, Shraman and Chellappa, Rama and Venugopalan, Subhashini},
+  journal={arXiv preprint arXiv:2407.09413},
+  year={2024}
+}
+```
+
+---
+
+## 🔗 Further References
+
+- 🤗 [Dataset on Hugging Face](https://huggingface.co/datasets/google/spiqa)  
+- 🐙 [GitHub Repository](https://github.com/google/spiqa)  
+- 📄 [SPIQA Paper (arXiv:2407.09413)](https://arxiv.org/pdf/2407.09413)
 
-## Limitations and Bias
-*Note any known limitations or biases that the metric has, with links and references if possible.*
 
-## Citation
-*Cite the source where this metric was introduced.*
 
-## Further References
-*Add any useful further references.*

app.py

@@ -2,5 +2,5 @@ import evaluate
 from evaluate.utils import launch_gradio_widget
 
 
-module = evaluate.load("nhop/l3score")
+module = evaluate.load("nhop/L3Score")
 launch_gradio_widget(module)

l3score.py

@@ -1,95 +0,0 @@
-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""TODO: Add a description here."""
-
-import evaluate
-import datasets
-
-
-# TODO: Add BibTeX citation
-_CITATION = """\
-@InProceedings{huggingface:module,
-title = {A great new module},
-authors={huggingface, Inc.},
-year={2020}
-}
-"""
-
-# TODO: Add description of the module here
-_DESCRIPTION = """\
-This new module is designed to solve this great ML task and is crafted with a lot of care.
-"""
-
-
-# TODO: Add description of the arguments of the module here
-_KWARGS_DESCRIPTION = """
-Calculates how good are predictions given some references, using certain scores
-Args:
-    predictions: list of predictions to score. Each predictions
-        should be a string with tokens separated by spaces.
-    references: list of reference for each prediction. Each
-        reference should be a string with tokens separated by spaces.
-Returns:
-    accuracy: description of the first score,
-    another_score: description of the second score,
-Examples:
-    Examples should be written in doctest format, and should illustrate how
-    to use the function.
-
-    >>> my_new_module = evaluate.load("my_new_module")
-    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
-    >>> print(results)
-    {'accuracy': 1.0}
-"""
-
-# TODO: Define external resources urls if needed
-BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
-
-
-@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
-class l3score(evaluate.Metric):
-    """TODO: Short description of my evaluation module."""
-
-    def _info(self):
-        # TODO: Specifies the evaluate.EvaluationModuleInfo object
-        return evaluate.MetricInfo(
-            # This is the description that will appear on the modules page.
-            module_type="metric",
-            description=_DESCRIPTION,
-            citation=_CITATION,
-            inputs_description=_KWARGS_DESCRIPTION,
-            # This defines the format of each prediction and reference
-            features=datasets.Features({
-                'predictions': datasets.Value('int64'),
-                'references': datasets.Value('int64'),
-            }),
-            # Homepage of the module for documentation
-            homepage="http://module.homepage",
-            # Additional links to the codebase or references
-            codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
-            reference_urls=["http://path.to.reference.url/new_module"]
-        )
-
-    def _download_and_prepare(self, dl_manager):
-        """Optional: download external resources useful to compute the scores"""
-        # TODO: Download external resources if needed
-        pass
-
-    def _compute(self, predictions, references):
-        """Returns the scores"""
-        # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
-        return {
-            "accuracy": accuracy,
-        }

requirements.txt

@@ -1 +1,7 @@
-git+https://github.com/huggingface/evaluate@main
+git+https://github.com/huggingface/evaluate@main
+langchain==0.3.23
+langchain-deepseek==0.1.3
+langchain-openai==0.3.12
+langchain-community==0.3.21
+langchain-core==0.3.52
+numpy==2.2.4