update

src/backend/run_eval_suite.py

@@ -5,10 +5,7 @@ from src.backend.manage_requests import EvalRequest
 
 from src.backend.tasks.xsum.task import XSum
 from src.backend.tasks.cnndm.task import CNNDM
-
-import logging
-
-logging.getLogger("openai").setLevel(logging.WARNING)
+from src.backend.tasks.selfcheckgpt.task import SelfCheckGpt
 
 
 def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, batch_size, device, use_cache=None, limit=None, max_nb_samples=100) -> dict:

src/backend/tasks/cnndm/task.py

@@ -141,6 +141,8 @@ class CNNDM(Task):
         # all_refs = true_refs + false_refs
 
         document = doc["article"]
+        gold_summary = doc["highlights"]
+
         true_refs = [doc["highlights"]]
         all_refs = true_refs
 

src/backend/tasks/selfcheckgpt/task.py

@@ -1,12 +1,13 @@
 import os
 from typing import Union, List
 
-
 from lm_eval.api.task import Task
 from lm_eval.api.instance import Instance
 from lm_eval.api.registry import register_task
 from lm_eval.api.metrics import mean
 
+from src.backend.envs import DEVICE
+
 import spacy
 from selfcheckgpt.modeling_selfcheck import SelfCheckMQAG, SelfCheckNLI, SelfCheckBERTScore, SelfCheckNgram
 
@@ -16,7 +17,7 @@ class SelfCheckGpt(Task):
     VERSION = 0.0
     DATASET_PATH = "potsawee/wiki_bio_gpt3_hallucination"
     DATASET_NAME = None
-    OUTPUT_TYPE = 'generate_until'
+
     def __init__(self, data_dir=None, cache_dir=None, download_mode=None, config=None):
         super().__init__(data_dir=data_dir, cache_dir=cache_dir, download_mode=download_mode, config=config)
         self.generation_kwargs = {"temperature": 0.0, "do_sample": False}
@@ -24,7 +25,7 @@ class SelfCheckGpt(Task):
         self.generation_kwargs_sampling = {"temperature": 1.0, "do_sample": False}
 
         self.selfcheckgpt_type = os.environ.get('SELFCHECKGPTTYPE', 'SelfCheckNgram')
-        self.selfcheckgpt_device = os.environ.get('SELFCHECKGPTDEVICE', 'cpu')
+        self.selfcheckgpt_device = os.environ.get('SELFCHECKGPTDEVICE', DEVICE)
         self.selfcheckgpt_nlp = spacy.load("en_core_web_sm")
 
         if self.selfcheckgpt_type == 'SelfCheckNgram':
@@ -59,34 +60,19 @@ class SelfCheckGpt(Task):
         answer = doc['wiki_bio_text']
         return answer
 
-    def construct_requests(
-        self, doc: dict, ctx: str, **kwargs
-    ) -> Union[List[Instance], Instance]:
+    def construct_requests(self, doc: dict, ctx: str, **kwargs) -> Union[List[Instance], Instance]:
         arguments = (ctx, self.generation_kwargs)
         request_list = [
-            Instance(
-                request_type=self.OUTPUT_TYPE,
-                doc=doc,
-                arguments=arguments,
-                idx=0,
-                **kwargs
-                ),
+            Instance(request_type='generate_until', doc=doc, arguments=arguments, idx=0, **kwargs),
         ]
         sampling_arguments = (ctx, self.generation_kwargs_sampling)
         request_list.extend([
-            Instance(
-                request_type=self.OUTPUT_TYPE,
-                doc=doc,
-                arguments=sampling_arguments,
-                idx=idx,
-                **kwargs
-                )
+            Instance(request_type='generate_until', doc=doc, arguments=sampling_arguments, idx=idx, **kwargs)
             for idx in range(1, self.generation_kwargs_sampling_number+1)
             ]
         )
         return request_list
 
-
     def process_results(self, doc, results):
         response_temperature_0 = results[0]
         other_responses = results[1:]
@@ -104,24 +90,14 @@ class SelfCheckGpt(Task):
                     'max-selfcheckgpt': selfcheckgpt_scores['doc_level']['avg_max_neg_logprob']}
 
         elif self.selfcheckgpt_type == 'SelfCheckBERTScore':
-            selfcheckgpt_scores = self.selfcheckgpt.predict(
-                sentences = sentences,
-                sampled_passages = other_responses,
-                )
+            selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, sampled_passages=other_responses)
         elif self.selfcheckgpt_type == 'SelfCheckMQAG':
-            selfcheckgpt_scores = self.selfcheckgpt.predict(
-                sentences = sentences,
-                sampled_passages = other_responses,
-                )
+            selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, sampled_passages=other_responses)
         elif self.selfcheckgpt_type == 'SelfCheckNLI':
-            selfcheckgpt_scores = self.selfcheckgpt.predict(
-                sentences = sentences,
-                passage = response_temperature_0,
-                sampled_passages = other_responses,
-                num_questions_per_sent = 5,          # number of questions to be drawn
-                scoring_method = 'bayes_with_alpha', # options = 'counting', 'bayes', 'bayes_with_alpha'
-                beta1 = 0.8, beta2 = 0.8,            # additional params depending on scoring_method
-                )
+            selfcheckgpt_scores = self.selfcheckgpt.predict(sentences=sentences, passage=response_temperature_0, sampled_passages=other_responses,
+                                                            num_questions_per_sent=5,           # number of questions to be drawn
+                                                            scoring_method='bayes_with_alpha',  # options = 'counting', 'bayes', 'bayes_with_alpha'
+                                                            beta1=0.8, beta2=0.8)               # additional params depending on scoring_method
 
         selfcheckgpt_scores_avg = sum(selfcheckgpt_scores) / len(selfcheckgpt_scores) if len(selfcheckgpt_scores) > 0 else 0
         selfcheckgpt_scores_max = max(selfcheckgpt_scores)