Upload 8 files

scripts/download_files.py

@@ -7,6 +7,9 @@ LOCAL_SAVE_PATH = "data"  # Path where files will be saved
 GITHUB_API_URL = "https://api.github.com/repos/chen700564/RGB/contents/data"
 RAW_BASE_URL = "https://raw.githubusercontent.com/chen700564/RGB/master/data/"
 
+# Ensure the directory exists before downloading
+os.makedirs(LOCAL_SAVE_PATH, exist_ok=True)
+
 def get_file_list():
     """Fetch the list of files from the GitHub repository."""
     response = requests.get(GITHUB_API_URL)
@@ -30,7 +33,6 @@ def download_file(file_name):
 
     file_url = RAW_BASE_URL + file_name
     local_file_path = os.path.join(LOCAL_SAVE_PATH, file_name)
-
     response = requests.get(file_url, stream=True)
     if response.status_code == 200:
         total_size = int(response.headers.get("content-length", 0))

scripts/evaluate_factual_robustness.py

@@ -9,15 +9,15 @@ from scripts.prompt import get_factual_prompt
 
 def evaluate_factual_robustness(config):
     """Evaluates negative rejection for a given model by processing predictions and computing scores."""
-    config["noise_rate"] = 0.4 # Time being to do clarification
-    modelname = config["model_name"]
-    noise_rate = config["noise_rate"]
-    passage_num = config["passage_num"]
+    config['noise_rate'] = 0.4 # Time being to do clarification
+    modelname = config['model_name']
+    noise_rate = config['noise_rate']
+    passage_num = config['passage_num']
     
-    if config["model_name"] in config["models"]:
-        model = GroqClient(plm=config["model_name"])
+    if config['model_name'] in config["models"]:
+        model = GroqClient(plm=config['model_name'])
     else:
-        logging.warning(f"Skipping unknown model: {config["model_name"]}")
+        logging.warning(f"Skipping unknown model: {config['model_name']}")
         return
 
     # File paths
@@ -84,7 +84,7 @@ def evaluate_factual_robustness(config):
             'rejecttt':rejecttt,
             'correct_tt':correct_tt,
             'nums': len(results),
-            'noise_rate': config["noise_rate"],
+            'noise_rate': config['noise_rate'],
         }
         return scores
     

scripts/evaluate_information_integration.py

@@ -11,11 +11,11 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 # Improved function to evaluate noise robustness
 def evaluate_information_integration(config):
     result_path = config["result_path"] + 'Information Integration/'
-    noise_rate = config["noise_rate"]
-    passage_num = config["passage_num"]
+    noise_rate = config['noise_rate']
+    passage_num = config['passage_num']
 
     # Iterate over each model specified in the config
-    filename = os.path.join(result_path, f'prediction_{config["model_name"]}_noise_{noise_rate}_passage_{passage_num}.json')
+    filename = os.path.join(result_path, f'prediction_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
     ensure_directory_exists(filename)
 
     # Load existing results if file exists
@@ -45,7 +45,7 @@ def evaluate_information_integration(config):
 
     # Save the final score file with tt and all_rate
     scores = {
-        'model': config["model_name"],
+        'model': config['model_name'],
         'accuracy': accuracy,
         'noise_rate': noise_rate,
         'correct_count': correct_count,
@@ -56,7 +56,7 @@ def evaluate_information_integration(config):
     logging.info(f"Score: {scores}")
     logging.info(f"Information Integration Accuracy: {accuracy:.2%}")
     
-    score_filename = os.path.join(result_path, f'scores_{config["model_name"]}_noise_{noise_rate}_passage_{passage_num}.json')
+    score_filename = os.path.join(result_path, f'scores_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
     with open(score_filename, 'w') as f:
         json.dump(scores, f, ensure_ascii=False, indent=4)
 

scripts/evaluate_negative_rejection.py

@@ -10,15 +10,15 @@ from scripts.prompt import get_prompt
 
 def evaluate_negative_rejection(config):
     """Evaluates negative rejection for a given model by processing predictions and computing scores."""
-    config["noise_rate"] = 1.0 # Noise rate should be 1.0 for negative rejection evaluation
-    modelname = config["model_name"]
-    noise_rate = config["noise_rate"]
-    passage_num = config["passage_num"]
+    config['noise_rate'] = 1.0 # Noise rate should be 1.0 for negative rejection evaluation
+    modelname = config['model_name']
+    noise_rate = config['noise_rate']
+    passage_num = config['passage_num']
     
-    if config["model_name"] in config["models"]:
-        model = GroqClient(plm=config["model_name"])
+    if config['model_name'] in config["models"]:
+        model = GroqClient(plm=config['model_name'])
     else:
-        logging.warning(f"Skipping unknown model: {config["model_name"]}")
+        logging.warning(f"Skipping unknown model: {config['model_name']}")
         return
 
     # File paths

scripts/evaluate_noise_robustness.py

@@ -11,11 +11,11 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 # Improved function to evaluate noise robustness
 def evaluate_noise_robustness(config):
     result_path = config["result_path"] + 'Noise Robustness/'
-    noise_rate = config["noise_rate"]
-    passage_num = config["passage_num"]
+    noise_rate = config['noise_rate']
+    passage_num = config['passage_num']
 
     # Iterate over each model specified in the config
-    filename = os.path.join(result_path, f'prediction_{config["model_name"]}_noise_{noise_rate}_passage_{passage_num}.json')
+    filename = os.path.join(result_path, f'prediction_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
     ensure_directory_exists(filename)
 
     # Load existing results if file exists
@@ -45,7 +45,7 @@ def evaluate_noise_robustness(config):
 
     # Save the final score file with tt and all_rate
     scores = {
-        'model': config["model_name"],
+        'model': config['model_name'],
         'accuracy': accuracy,
         'noise_rate': noise_rate,
         'correct_count': correct_count,
@@ -56,7 +56,7 @@ def evaluate_noise_robustness(config):
     logging.info(f"score: {scores}")
     logging.info(f"Noise Robustness Accuracy: {accuracy:.2%}")
     
-    score_filename = os.path.join(result_path, f'scores_{config["model_name"]}_noise_{noise_rate}_passage_{passage_num}.json')
+    score_filename = os.path.join(result_path, f'scores_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
     with open(score_filename, 'w') as f:
         json.dump(scores, f, ensure_ascii=False, indent=4)
 

scripts/get_factual_evaluation.py

@@ -11,11 +11,11 @@ logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(
 # Improved function to evaluate noise robustness
 def get_factual_evaluation(config):
     result_path = config["result_path"] + 'Counterfactual Robustness/'
-    noise_rate = config["noise_rate"]
-    passage_num = config["passage_num"]
+    noise_rate = config['noise_rate']
+    passage_num = config['passage_num']
 
     # Iterate over each model specified in the config
-    filename = os.path.join(result_path, f'prediction_{config["model_name"]}_noise_{noise_rate}_passage_{passage_num}.json')
+    filename = os.path.join(result_path, f'prediction_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
     ensure_directory_exists(filename)
 
     # Load existing results if file exists
@@ -61,7 +61,7 @@ def get_factual_evaluation(config):
     scores['correct_tt'] = correct_tt
 
     #logging.info(f"score: {scores}")
-    score_filename = os.path.join(result_path, f'scores_{config["model_name"]}_noise_{noise_rate}_passage_{passage_num}.json')
+    score_filename = os.path.join(result_path, f'scores_{config['model_name']}_noise_{noise_rate}_passage_{passage_num}.json')
     with open(score_filename, 'w') as f:
         json.dump(scores, f, ensure_ascii=False, indent=4)
 

scripts/get_prediction_result.py

@@ -13,17 +13,17 @@ def get_prediction_result(config, data_file_name):
     results = []
     dataset = load_dataset(data_file_name)
     # Create GroqClient instance for supported models
-    if config["model_name"] in config["models"]:
-        model = GroqClient(plm=config["model_name"])
+    if config['model_name'] in config["models"]:
+        model = GroqClient(plm=config['model_name'])
     else:
-        logging.warning(f"Skipping unknown model: {config["model_name"]}")
+        logging.warning(f"Skipping unknown model: {config['model_name']}")
         return
     
     # Iterate through dataset and process queries
-    for idx, instance in enumerate(dataset[:config["num_queries"]], start=0):
-        logging.info(f"Executing Query {idx + 1} for Model: {config["model_name"]}")
+    for idx, instance in enumerate(dataset[:config['num_queries']], start=0):
+        logging.info(f"Executing Query {idx + 1} for Model: {config['model_name']}")
 
-        query, ans, docs = process_data(instance, config["noise_rate"], config["passage_num"], data_file_name)
+        query, ans, docs = process_data(instance, config['noise_rate'], config['passage_num'], data_file_name)
 
         # Retry mechanism for prediction
         for attempt in range(1, config["retry_attempts"] + 1):
@@ -46,7 +46,7 @@ def get_prediction_result(config, data_file_name):
             'label': label,
             'prediction': prediction,
             'docs': docs,
-            'noise_rate': config["noise_rate"],
+            'noise_rate': config['noise_rate'],
             'factlabel': factlabel
         }
         results.append(new_instance)

scripts/helper.py

@@ -31,11 +31,11 @@ def update_config(config, model_name=None, noise_rate=None, num_queries=None):
         dict: The updated configuration dictionary.
     """
     if model_name:
-        config["model_name"] = model_name
+        config['model_name'] = model_name
     if noise_rate is not None:  # Explicitly check for None to handle 0.0
-        config["noise_rate"] = float(noise_rate)  # Ensure it's a float
+        config['noise_rate'] = float(noise_rate)  # Ensure it's a float
     if num_queries is not None:  # Explicitly check for None to handle 0
-        config["num_queries"] = int(num_queries)  # Ensure it's an integer
+        config['num_queries'] = int(num_queries)  # Ensure it's an integer
     return config
 
 def load_dataset(file_name):