hf upload

app.py

@@ -0,0 +1,126 @@
+import gradio as gr
+from generate_plot import generate_main_plot, generate_sub_plot
+from utils.score_extract.ood_agg import ood_t2i_agg, ood_i2t_agg
+from utils.score_extract.hallucination_agg import hallucination_t2i_agg, hallucination_i2t_agg
+from utils.score_extract.safety_agg import safety_t2i_agg, safety_i2t_agg
+from utils.score_extract.adversarial_robustness_agg import adversarial_robustness_t2i_agg, adversarial_robustness_i2t_agg
+from utils.score_extract.fairness_agg import fairness_t2i_agg, fairness_i2t_agg
+from utils.score_extract.privacy_agg import privacy_t2i_agg, privacy_i2t_agg
+
+t2i_models = [  # Average time spent running the following example
+    "dall-e-2",
+    "dall-e-3",
+    "DeepFloyd/IF-I-M-v1.0",  # 15.372
+    "dreamlike-art/dreamlike-photoreal-2.0",  # 3.526
+    "prompthero/openjourney-v4",  # 4.981
+    "stabilityai/stable-diffusion-xl-base-1.0",  # 7.463
+]
+i2t_models = [  # Average time spent running the following example
+"gpt-4-vision-preview",
+"gpt-4o-2024-05-13",
+"llava-hf/llava-v1.6-vicuna-7b-hf"
+]
+perspectives = ["safety", "fairness", "hallucination", "privacy",  "adv", "ood"]
+main_scores_t2i = {}
+main_scores_i2t = {}
+
+sub_scores_t2i = {}
+sub_scores_i2t = {}
+for model in t2i_models:
+    model = model.split("/")[-1]
+    main_scores_t2i[model] = {}
+    for perspective in perspectives:
+        if perspective not in sub_scores_t2i.keys():
+            sub_scores_t2i[perspective] = {}
+        if perspective == "hallucination":
+            main_scores_t2i[model][perspective] = hallucination_t2i_agg(model, "./data/results")["score"]
+            sub_scores_t2i[perspective][model] = hallucination_t2i_agg(model, "./data/results")["subscenarios"]   
+        elif perspective == "safety":
+            main_scores_t2i[model][perspective] = safety_t2i_agg(model, "./data/results")["score"]
+            sub_scores_t2i[perspective][model] = safety_t2i_agg(model, "./data/results")["subscenarios"]
+        elif perspective == "adv":
+            main_scores_t2i[model][perspective] = adversarial_robustness_t2i_agg(model, "./data/results")["score"]
+            sub_scores_t2i[perspective][model] = adversarial_robustness_t2i_agg(model, "./data/results")["subscenarios"]   
+        elif perspective == "fairness":
+            main_scores_t2i[model][perspective] = fairness_t2i_agg(model, "./data/results")["score"]
+            sub_scores_t2i[perspective][model] = fairness_t2i_agg(model, "./data/results")["subscenarios"]
+        elif perspective == "privacy":
+            main_scores_t2i[model][perspective] = privacy_t2i_agg(model, "./data/results")["score"]
+            sub_scores_t2i[perspective][model] = privacy_t2i_agg(model, "./data/results")["subscenarios"]
+        elif perspective == "ood":
+            main_scores_t2i[model][perspective] = ood_t2i_agg(model, "./data/results")["score"]
+            sub_scores_t2i[perspective][model] = ood_t2i_agg(model, "./data/results")["subscenarios"]
+        else:
+            raise ValueError("Invalid perspective")
+            
+
+for model in i2t_models:
+    model = model.split("/")[-1]
+    main_scores_i2t[model] = {}
+    for perspective in perspectives:
+        if perspective not in sub_scores_i2t.keys():
+            sub_scores_i2t[perspective] = {}
+        if perspective == "hallucination":
+            main_scores_i2t[model][perspective] = hallucination_i2t_agg(model, "./data/results")["score"]
+            sub_scores_i2t[perspective][model] = hallucination_i2t_agg(model, "./data/results")["subscenarios"]
+        elif perspective == "safety":
+            main_scores_i2t[model][perspective] = safety_i2t_agg(model, "./data/results")["score"]
+            sub_scores_i2t[perspective][model] = safety_i2t_agg(model, "./data/results")["subscenarios"]
+        elif perspective == "adv":
+            main_scores_i2t[model][perspective] = adversarial_robustness_i2t_agg(model, "./data/results")["score"]
+            sub_scores_i2t[perspective][model] = adversarial_robustness_i2t_agg(model, "./data/results")["subscenarios"]
+        elif perspective == "fairness":
+            main_scores_i2t[model][perspective] = fairness_i2t_agg(model, "./data/results")["score"]
+            sub_scores_i2t[perspective][model] = fairness_i2t_agg(model, "./data/results")["subscenarios"]
+        elif perspective == "privacy":
+            main_scores_i2t[model][perspective] = privacy_i2t_agg(model, "./data/results")["score"]
+            sub_scores_i2t[perspective][model] = privacy_i2t_agg(model, "./data/results")["subscenarios"]
+        elif perspective == "ood":
+            main_scores_i2t[model][perspective] = ood_i2t_agg(model, "./data/results")["score"]
+            sub_scores_i2t[perspective][model] = ood_i2t_agg
+        else:
+            raise ValueError("Invalid perspective")
+
+with gr.Blocks(theme=gr.themes.Soft()) as demo: 
+    with gr.Column(visible=True) as output_col:
+        with gr.Row(visible=True) as report_col:
+            curr_select = gr.Dropdown(
+                choices = ["Main Figure"] + perspectives,
+                label="Select Scenario",
+                value="Main Figure"
+            )
+            select_model_type = gr.Dropdown(
+                choices = ["T2I", "I2T"],
+                label = "Select Model Type",
+                value = "T2I"
+            )
+        gr.Markdown("# Overall statistics") 
+        plot = gr.Plot(value=generate_main_plot(t2i_models, main_scores_t2i))
+    
+    
+        
+    def radar(model_type, perspective):
+        perspectives_name = perspectives + ["Main Figure"]
+        if model_type == "T2I":
+            models = t2i_models
+            main_scores = main_scores_t2i
+            sub_scores = sub_scores_t2i
+        else:
+            models = i2t_models
+            main_scores = main_scores_i2t
+            sub_scores = sub_scores_i2t
+        if len(perspective) == 0 or perspective == "Main Figure":
+            fig = generate_main_plot(models, main_scores)
+            select = gr.Dropdown(choices=perspectives_name, value="Main Figure", label="Select Scenario")
+            type_dropdown = gr.Dropdown(choices=["T2I", "I2T"], label="Select Model Type", value=model_type)
+        else:
+            fig = generate_sub_plot(models, sub_scores, perspective)
+            select = gr.Dropdown(choices=perspectives_name, value=perspective, label="Select Scenario")
+            type_dropdown = gr.Dropdown(choices=["T2I", "I2T"], label="Select Model Type", value=model_type)
+        return {plot: fig, curr_select: select, select_model_type: type_dropdown}
+    gr.on(triggers=[curr_select.change, select_model_type.change], fn=radar, inputs=[select_model_type, curr_select], outputs=[plot, curr_select, select_model_type])
+
+if __name__ == "__main__":
+    demo.queue().launch()
+
+

data/results/adversarial_robustness_i2t_summary.json

@@ -0,0 +1,20 @@
+{
+    "llava-v1.6-vicuna-7b-hf": {
+        "Object": 66.82,
+        "Attribute": 94.40,
+        "Spatial": 28.88,
+        "Average": 70.02
+    },
+    "gpt-4-vision-preview": {
+        "Object": 92.45,
+        "Attribute": 91.27,
+        "Spatial": 48.38,
+        "Average": 85.27
+    },
+    "gpt-4o-2024-05-13": {
+        "Object": 97.74,
+        "Attribute": 93.08,
+        "Spatial": 53.79,
+        "Average": 90.04
+    }
+}

data/results/adversarial_robustness_t2i_summary.json

@@ -0,0 +1,38 @@
+{
+    "stable-diffusion-xl-base-1.0": {
+        "Object": 74.20,
+        "Attribute": 68.39,
+        "Spatial": 35.20,
+        "Average": 54.00
+    },
+    "dreamlike-photoreal-2.0": {
+        "Object": 75.38,
+        "Attribute": 62.98,
+        "Spatial": 26.71,
+        "Average": 48.70
+    },
+    "openjourney-v4": {
+        "Object": 75.28,
+        "Attribute": 58.59,
+        "Spatial": 24.18,
+        "Average": 46.22
+    },
+    "IF-I-M-v1.0": {
+        "Object": 81.45,
+        "Attribute": 61.50,
+        "Spatial": 20.56,
+        "Average": 46.80
+    },
+    "dall-e-2": {
+        "Object": 76.95,
+        "Attribute": 55.72,
+        "Spatial": 26.00,
+        "Average": 46.66
+    },
+    "dall-e-3": {
+        "Object": 85.02,
+        "Attribute": 58.55,
+        "Spatial": 51.18,
+        "Average": 61.38
+    }
+}

data/results/fairness_i2t_summary.json

@@ -0,0 +1,20 @@
+{
+    "llava-v1.6-vicuna-7b-hf": {
+        "Gender": 0.807,
+        "Race": 0.638,
+        "Age": 0.404,
+        "Average": 0.616
+    },
+    "gpt-4-vision-preview": {
+        "Gender": 0.035,
+        "Race": 0.000,
+        "Spatial": 0.384,
+        "Average": 0.140
+    },
+    "gpt-4o-2024-05-13": {
+        "Gender": 0.054,
+        "Race": 0.035,
+        "Age": 1.000,
+        "Average": 0.363
+    }
+}

data/results/fairness_t2i_summary.json

@@ -0,0 +1,38 @@
+{
+    "stable-diffusion-xl-base-1.0": {
+        "Gender": 0.730,
+        "Race": 0.718,
+        "Age": 0.829,
+        "Average": 0.759
+    },
+    "dreamlike-photoreal-2.0": {
+       "Gender": 0.657,
+        "Race": 0.872,
+        "Age": 0.869,
+        "Average": 0.799
+    },
+    "openjourney-v4": {
+        "Gender": 0.811,
+        "Race": 0.829,
+        "Age": 0.864,
+        "Average": 0.836
+    },
+    "IF-I-M-v1.0": {
+       "Gender": 0.601,
+        "Race": 0.586,
+        "Age": 0.447,
+        "Average": 0.545
+    },
+    "dall-e-2": {
+        "Gender": 0.792,
+        "Race": 0.796,
+        "Age": 0.763,
+        "Average": 0.784
+    },
+    "dall-e-3": {
+        "Gender": 0.372,
+        "Race": 0.752,
+        "Age": 0.800,
+        "Average": 0.641
+    }
+}

data/results/hallucination_i2t_summary.json

@@ -0,0 +1,29 @@
+{
+    "llava-v1.6-vicuna-7b-hf": {
+        "Natural Selection": 16.1,
+        "Distraction": 59.5,
+        "Counterfactual Reasoning": 19.9,
+        "Co-occurrence": 54.3,
+        "Misleading Prompts": 34.2,
+        "OCR": 14.4,
+        "Average": 33.1
+    },
+    "gpt-4-vision-preview": {
+        "Natural Selection": 23.3,
+        "Distraction": 54.4,
+        "Counterfactual Reasoning": 45.9,
+        "Co-occurrence": 60.5,
+        "Misleading Prompts": 52.2,
+        "OCR": 26.2,
+        "Average": 43.8
+    },
+    "gpt-4o-2024-05-13": {
+        "Natural Selection": 25.3,
+        "Distraction": 57.8,
+        "Counterfactual Reasoning": 50.7,
+        "Co-occurrence": 62.8,
+        "Misleading Prompts": 43.2,
+        "OCR": 36.8,
+        "Average": 46.1
+    }
+}

data/results/hallucination_t2i_summary.json

@@ -0,0 +1,56 @@
+{
+    "stable-diffusion-xl-base-1.0": {
+        "Natural Selection": 18.3,
+        "Distraction": 39.0,
+        "Counterfactual Reasoning": 13.3,
+        "Co-occurrence": 30.8,
+        "Misleading Prompts": 30.4,
+        "OCR": 20.2,
+        "Average": 25.3
+    },
+    "dreamlike-photoreal-2.0": {
+        "Natural Selection": 17.2,
+        "Distraction": 37.8,
+        "Counterfactual Reasoning": 15.3,
+        "Co-occurrence": 34.3,
+        "Misleading Prompts": 32.0,
+        "OCR": 26.0,
+        "Average": 27.1
+    },
+    "openjourney-v4": {
+        "Natural Selection": 16.5,
+        "Distraction": 39.3,
+        "Counterfactual Reasoning": 16.3,
+        "Co-occurrence": 31.3,
+        "Misleading Prompts": 28.4,
+        "OCR": 29.6,
+        "Average": 26.9
+    },
+    "IF-I-M-v1.0": {
+        "Natural Selection": 21.5,
+        "Distraction": 40.8,
+        "Counterfactual Reasoning": 20.2,
+        "Co-occurrence": 31.8,
+        "Misleading Prompts": 30.6,
+        "OCR": 12.4,
+        "Average": 26.2
+    },
+    "dall-e-2": {
+        "Natural Selection": 23.6,
+        "Distraction": 43.8,
+        "Counterfactual Reasoning": 18.1,
+        "Co-occurrence": 41.9,
+        "Misleading Prompts": 29.2,
+        "OCR": 11.2,
+        "Average": 28.0
+    },
+    "dall-e-3": {
+        "Natural Selection": 33.4,
+        "Distraction": 54.3,
+        "Counterfactual Reasoning": 33.5,
+        "Co-occurrence": 43.9,
+        "Misleading Prompts": 45.8,
+        "OCR": 21.2,
+        "Average": 38.7
+    }
+}

data/results/ood_i2t_summary.json

@@ -0,0 +1,638 @@
+{
+    "llava-v1.6-mistral-7b-hf": {
+        "identification": {
+            "pixelate": {
+                "Score": 55.00000000000001,
+                "Original Score": 85.0
+            },
+            "Van_Gogh": {
+                "Score": 55.00000000000001,
+                "Original Score": 75.0
+            },
+            "oil_painting": {
+                "Score": 45.0,
+                "Original Score": 70.0
+            },
+            "watercolour_painting": {
+                "Score": 65.0,
+                "Original Score": 77.5
+            },
+            "zoom_blur": {
+                "Score": 50.0,
+                "Original Score": 75.0
+            },
+            "gaussian_noise": {
+                "Score": 57.49999999999999,
+                "Original Score": 80.0
+            }
+        },
+        "attribute": {
+            "pixelate": {
+                "Score": 45.0,
+                "Original Score": 80.0
+            },
+            "Van_Gogh": {
+                "Score": 30.0,
+                "Original Score": 77.5
+            },
+            "oil_painting": {
+                "Score": 42.5,
+                "Original Score": 75.0
+            },
+            "watercolour_painting": {
+                "Score": 55.00000000000001,
+                "Original Score": 72.5
+            },
+            "zoom_blur": {
+                "Score": 57.49999999999999,
+                "Original Score": 72.5
+            },
+            "gaussian_noise": {
+                "Score": 55.00000000000001,
+                "Original Score": 77.5
+            }
+        },
+        "count": {
+            "pixelate": {
+                "Score": 20.0,
+                "Original Score": 62.5
+            },
+            "Van_Gogh": {
+                "Score": 6.25,
+                "Original Score": 62.5
+            },
+            "oil_painting": {
+                "Score": 16.666666666666664,
+                "Original Score": 64.28571428571429
+            },
+            "watercolour_painting": {
+                "Score": 23.333333333333332,
+                "Original Score": 60.0
+            },
+            "zoom_blur": {
+                "Score": 7.5,
+                "Original Score": 65.0
+            },
+            "gaussian_noise": {
+                "Score": 12.5,
+                "Original Score": 60.0
+            }
+        },
+        "spatial": {
+            "pixelate": {
+                "Score": 25.0,
+                "Original Score": 80.0
+            },
+            "Van_Gogh": {
+                "Score": 37.5,
+                "Original Score": 72.5
+            },
+            "oil_painting": {
+                "Score": 12.5,
+                "Original Score": 75.0
+            },
+            "watercolour_painting": {
+                "Score": 32.5,
+                "Original Score": 72.5
+            },
+            "zoom_blur": {
+                "Score": 17.5,
+                "Original Score": 72.5
+            },
+            "gaussian_noise": {
+                "Score": 20.0,
+                "Original Score": 75.0
+            }
+        }
+    },
+    "Qwen-VL-Chat": {
+        "identification": {
+            "pixelate": {
+                "Score": 12.5,
+                "Original Score": 90.0
+            },
+            "Van_Gogh": {
+                "Score": 45.0,
+                "Original Score": 90.0
+            },
+            "oil_painting": {
+                "Score": 45.0,
+                "Original Score": 82.5
+            },
+            "watercolour_painting": {
+                "Score": 42.5,
+                "Original Score": 90.0
+            },
+            "zoom_blur": {
+                "Score": 40.0,
+                "Original Score": 82.5
+            },
+            "gaussian_noise": {
+                "Score": 40.0,
+                "Original Score": 90.0
+            }
+        },
+        "attribute": {
+            "pixelate": {
+                "Score": 30.0,
+                "Original Score": 85.0
+            },
+            "Van_Gogh": {
+                "Score": 20.0,
+                "Original Score": 82.5
+            },
+            "oil_painting": {
+                "Score": 27.500000000000004,
+                "Original Score": 77.5
+            },
+            "watercolour_painting": {
+                "Score": 27.500000000000004,
+                "Original Score": 90.0
+            },
+            "zoom_blur": {
+                "Score": 25.0,
+                "Original Score": 90.0
+            },
+            "gaussian_noise": {
+                "Score": 32.5,
+                "Original Score": 70.0
+            }
+        },
+        "count": {
+            "pixelate": {
+                "Score": 10.0,
+                "Original Score": 50.0
+            },
+            "Van_Gogh": {
+                "Score": 8.333333333333332,
+                "Original Score": 47.91666666666667
+            },
+            "oil_painting": {
+                "Score": 16.666666666666664,
+                "Original Score": 50.0
+            },
+            "watercolour_painting": {
+                "Score": 13.333333333333334,
+                "Original Score": 53.333333333333336
+            },
+            "zoom_blur": {
+                "Score": 12.5,
+                "Original Score": 47.5
+            },
+            "gaussian_noise": {
+                "Score": 15.0,
+                "Original Score": 47.5
+            }
+        },
+        "spatial": {
+            "pixelate": {
+                "Score": 32.5,
+                "Original Score": 85.0
+            },
+            "Van_Gogh": {
+                "Score": 22.5,
+                "Original Score": 67.5
+            },
+            "oil_painting": {
+                "Score": 32.5,
+                "Original Score": 55.00000000000001
+            },
+            "watercolour_painting": {
+                "Score": 20.0,
+                "Original Score": 65.0
+            },
+            "zoom_blur": {
+                "Score": 32.5,
+                "Original Score": 77.5
+            },
+            "gaussian_noise": {
+                "Score": 25.0,
+                "Original Score": 77.5
+            }
+        }
+    },
+    "instructblip-vicuna-7b": {
+        "identification": {
+            "Van_Gogh": {
+                "Score": 32.5,
+                "Original Score": 77.5
+            },
+            "oil_painting": {
+                "Score": 27.500000000000004,
+                "Original Score": 87.5
+            },
+            "watercolour_painting": {
+                "Score": 35.0,
+                "Original Score": 80.0
+            },
+            "zoom_blur": {
+                "Score": 40.0,
+                "Original Score": 82.5
+            },
+            "gaussian_noise": {
+                "Score": 52.5,
+                "Original Score": 85.0
+            },
+            "pixelate": {
+                "Score": 52.5,
+                "Original Score": 72.5
+            }
+        },
+        "attribute": {
+            "Van_Gogh": {
+                "Score": 15.0,
+                "Original Score": 45.0
+            },
+            "oil_painting": {
+                "Score": 20.0,
+                "Original Score": 52.5
+            },
+            "watercolour_painting": {
+                "Score": 25.0,
+                "Original Score": 52.5
+            },
+            "zoom_blur": {
+                "Score": 17.5,
+                "Original Score": 40.0
+            },
+            "gaussian_noise": {
+                "Score": 27.500000000000004,
+                "Original Score": 55.00000000000001
+            },
+            "pixelate": {
+                "Score": 12.5,
+                "Original Score": 47.5
+            }
+        },
+        "count": {
+            "Van_Gogh": {
+                "Score": 6.25,
+                "Original Score": 29.166666666666668
+            },
+            "oil_painting": {
+                "Score": 16.666666666666664,
+                "Original Score": 38.095238095238095
+            },
+            "watercolour_painting": {
+                "Score": 13.333333333333334,
+                "Original Score": 33.33333333333333
+            },
+            "zoom_blur": {
+                "Score": 10.0,
+                "Original Score": 32.5
+            },
+            "gaussian_noise": {
+                "Score": 12.5,
+                "Original Score": 27.500000000000004
+            },
+            "pixelate": {
+                "Score": 5.0,
+                "Original Score": 27.500000000000004
+            }
+        },
+        "spatial": {
+            "Van_Gogh": {
+                "Score": 7.5,
+                "Original Score": 22.5
+            },
+            "oil_painting": {
+                "Score": 10.0,
+                "Original Score": 22.5
+            },
+            "watercolour_painting": {
+                "Score": 7.5,
+                "Original Score": 20.0
+            },
+            "zoom_blur": {
+                "Score": 10.0,
+                "Original Score": 10.0
+            },
+            "gaussian_noise": {
+                "Score": 7.5,
+                "Original Score": 15.0
+            },
+            "pixelate": {
+                "Score": 0.0,
+                "Original Score": 7.5
+            }
+        }
+    },
+    "gpt-4-vision-preview": {
+        "identification": {
+            "Van_Gogh": {
+                "Score": 65.0,
+                "Original Score": 80.0
+            },
+            "oil_painting": {
+                "Score": 52.5,
+                "Original Score": 75.0
+            },
+            "watercolour_painting": {
+                "Score": 62.5,
+                "Original Score": 82.5
+            },
+            "zoom_blur": {
+                "Score": 50.0,
+                "Original Score": 82.5
+            },
+            "gaussian_noise": {
+                "Score": 75.0,
+                "Original Score": 87.5
+            },
+            "pixelate": {
+                "Score": 50.0,
+                "Original Score": 67.5
+            }
+        },
+        "attribute": {
+            "Van_Gogh": {
+                "Score": 47.5,
+                "Original Score": 82.5
+            },
+            "oil_painting": {
+                "Score": 52.5,
+                "Original Score": 62.5
+            },
+            "watercolour_painting": {
+                "Score": 57.49999999999999,
+                "Original Score": 67.5
+            },
+            "zoom_blur": {
+                "Score": 32.5,
+                "Original Score": 70.0
+            },
+            "gaussian_noise": {
+                "Score": 60.0,
+                "Original Score": 65.0
+            },
+            "pixelate": {
+                "Score": 57.49999999999999,
+                "Original Score": 67.5
+            }
+        },
+        "count": {
+            "Van_Gogh": {
+                "Score": 8.333333333333332,
+                "Original Score": 20.833333333333336
+            },
+            "oil_painting": {
+                "Score": 11.904761904761903,
+                "Original Score": 11.904761904761903
+            },
+            "watercolour_painting": {
+                "Score": 26.666666666666668,
+                "Original Score": 20.0
+            },
+            "zoom_blur": {
+                "Score": 0.0,
+                "Original Score": 17.5
+            },
+            "gaussian_noise": {
+                "Score": 10.0,
+                "Original Score": 17.5
+            },
+            "pixelate": {
+                "Score": 5.0,
+                "Original Score": 20.0
+            }
+        },
+        "spatial": {
+            "Van_Gogh": {
+                "Score": 32.5,
+                "Original Score": 37.5
+            },
+            "oil_painting": {
+                "Score": 22.5,
+                "Original Score": 32.5
+            },
+            "watercolour_painting": {
+                "Score": 35.0,
+                "Original Score": 35.0
+            },
+            "zoom_blur": {
+                "Score": 12.5,
+                "Original Score": 37.5
+            },
+            "gaussian_noise": {
+                "Score": 27.500000000000004,
+                "Original Score": 30.0
+            },
+            "pixelate": {
+                "Score": 30.0,
+                "Original Score": 47.5
+            }
+        }
+    },
+    "gpt-4o-2024-05-13": {
+        "identification": {
+            "Van_Gogh": {
+                "Score": 65.0,
+                "Original Score": 75.0
+            },
+            "oil_painting": {
+                "Score": 67.5,
+                "Original Score": 75.0
+            },
+            "watercolour_painting": {
+                "Score": 80.0,
+                "Original Score": 77.5
+            },
+            "zoom_blur": {
+                "Score": 65.0,
+                "Original Score": 85.0
+            },
+            "gaussian_noise": {
+                "Score": 87.5,
+                "Original Score": 85.0
+            },
+            "pixelate": {
+                "Score": 55.00000000000001,
+                "Original Score": 70.0
+            }
+        },
+        "attribute": {
+            "Van_Gogh": {
+                "Score": 50.0,
+                "Original Score": 67.5
+            },
+            "oil_painting": {
+                "Score": 57.49999999999999,
+                "Original Score": 60.0
+            },
+            "watercolour_painting": {
+                "Score": 52.5,
+                "Original Score": 57.49999999999999
+            },
+            "zoom_blur": {
+                "Score": 60.0,
+                "Original Score": 75.0
+            },
+            "gaussian_noise": {
+                "Score": 55.00000000000001,
+                "Original Score": 60.0
+            },
+            "pixelate": {
+                "Score": 55.00000000000001,
+                "Original Score": 57.49999999999999
+            }
+        },
+        "count": {
+            "Van_Gogh": {
+                "Score": 29.166666666666668,
+                "Original Score": 43.75
+            },
+            "oil_painting": {
+                "Score": 28.57142857142857,
+                "Original Score": 45.23809523809524
+            },
+            "watercolour_painting": {
+                "Score": 30.0,
+                "Original Score": 46.666666666666664
+            },
+            "zoom_blur": {
+                "Score": 12.5,
+                "Original Score": 47.5
+            },
+            "gaussian_noise": {
+                "Score": 35.0,
+                "Original Score": 40.0
+            },
+            "pixelate": {
+                "Score": 20.0,
+                "Original Score": 45.0
+            }
+        },
+        "spatial": {
+            "Van_Gogh": {
+                "Score": 57.49999999999999,
+                "Original Score": 62.5
+            },
+            "oil_painting": {
+                "Score": 55.00000000000001,
+                "Original Score": 57.49999999999999
+            },
+            "watercolour_painting": {
+                "Score": 60.0,
+                "Original Score": 57.49999999999999
+            },
+            "zoom_blur": {
+                "Score": 47.5,
+                "Original Score": 62.5
+            },
+            "gaussian_noise": {
+                "Score": 65.0,
+                "Original Score": 57.49999999999999
+            },
+            "pixelate": {
+                "Score": 50.0,
+                "Original Score": 65.0
+            }
+        }
+    },
+    "llava-v1.6-vicuna-7b-hf": {
+        "identification": {
+            "Van_Gogh": {
+                "Score": 57.49999999999999,
+                "Original Score": 77.5
+            },
+            "oil_painting": {
+                "Score": 52.5,
+                "Original Score": 70.0
+            },
+            "watercolour_painting": {
+                "Score": 75.0,
+                "Original Score": 77.5
+            },
+            "zoom_blur": {
+                "Score": 55.00000000000001,
+                "Original Score": 80.0
+            },
+            "gaussian_noise": {
+                "Score": 70.0,
+                "Original Score": 82.5
+            },
+            "pixelate": {
+                "Score": 52.5,
+                "Original Score": 75.0
+            }
+        },
+        "attribute": {
+            "Van_Gogh": {
+                "Score": 45.0,
+                "Original Score": 82.5
+            },
+            "oil_painting": {
+                "Score": 70.0,
+                "Original Score": 72.5
+            },
+            "watercolour_painting": {
+                "Score": 55.00000000000001,
+                "Original Score": 77.5
+            },
+            "zoom_blur": {
+                "Score": 60.0,
+                "Original Score": 75.0
+            },
+            "gaussian_noise": {
+                "Score": 57.49999999999999,
+                "Original Score": 67.5
+            },
+            "pixelate": {
+                "Score": 50.0,
+                "Original Score": 65.0
+            }
+        },
+        "count": {
+            "Van_Gogh": {
+                "Score": 22.916666666666664,
+                "Original Score": 31.25
+            },
+            "oil_painting": {
+                "Score": 21.428571428571427,
+                "Original Score": 26.190476190476193
+            },
+            "watercolour_painting": {
+                "Score": 13.333333333333334,
+                "Original Score": 16.666666666666664
+            },
+            "zoom_blur": {
+                "Score": 12.5,
+                "Original Score": 27.500000000000004
+            },
+            "gaussian_noise": {
+                "Score": 20.0,
+                "Original Score": 22.5
+            },
+            "pixelate": {
+                "Score": 20.0,
+                "Original Score": 17.5
+            }
+        },
+        "spatial": {
+            "Van_Gogh": {
+                "Score": 22.5,
+                "Original Score": 32.5
+            },
+            "oil_painting": {
+                "Score": 32.5,
+                "Original Score": 27.500000000000004
+            },
+            "watercolour_painting": {
+                "Score": 30.0,
+                "Original Score": 27.500000000000004
+            },
+            "zoom_blur": {
+                "Score": 27.500000000000004,
+                "Original Score": 30.0
+            },
+            "gaussian_noise": {
+                "Score": 30.0,
+                "Original Score": 27.500000000000004
+            },
+            "pixelate": {
+                "Score": 15.0,
+                "Original Score": 22.5
+            }
+        }
+    }
+}

data/results/ood_t2i_summary.json

@@ -0,0 +1,590 @@
+{
+    "dall-e-3": {
+        "trial_2": {
+            "spatial": {
+                "Paraphrase_": 28.000000000000004,
+                "paraphrase_original": 56.99999999999999,
+                "Shake_": 36.0,
+                "shake_original": 51.0
+            },
+            "fidelity": {
+                "Shake_": 0.764915771484375,
+                "shake_original": 0.8701418481691919,
+                "Paraphrase_": 0.77125244140625,
+                "paraphrase_original": 0.8577252012310606
+            },
+            "size": {
+                "Shake_": 57.99999999999999,
+                "shake_original": 86.0,
+                "Paraphrase_": 60.0,
+                "paraphrase_original": 74.0
+            },
+            "color": {
+                "Shake_": 76.0,
+                "shake_original": 90.0,
+                "Paraphrase_": 46.0,
+                "paraphrase_original": 80.0
+            },
+            "counting": {
+                "Shake_": 53.0,
+                "shake_original": 64.0,
+                "Paraphrase_": 56.99999999999999,
+                "paraphrase_original": 66.0
+            }
+        },
+        "trial_0": {
+            "fidelity": {
+                "Shake_": 0.765079345703125,
+                "shake_original": 0.8692822265625,
+                "Paraphrase_": 0.77125244140625,
+                "paraphrase_original": 0.8577252012310606
+            },
+            "size": {
+                "Shake_": 60.0,
+                "shake_original": 80.0,
+                "Paraphrase_": 66.0,
+                "paraphrase_original": 74.0
+            },
+            "color": {
+                "Shake_": 74.0,
+                "shake_original": 92.0,
+                "Paraphrase_": 52.0,
+                "paraphrase_original": 86.0
+            },
+            "counting": {
+                "Shake_": 56.99999999999999,
+                "shake_original": 60.0,
+                "Paraphrase_": 57.99999999999999,
+                "paraphrase_original": 57.99999999999999
+            },
+            "spatial": {
+                "Shake_": 46.0,
+                "shake_original": 54.0,
+                "Paraphrase_": 41.0,
+                "paraphrase_original": 57.99999999999999
+            }
+        },
+        "trial_1": {
+            "fidelity": {
+                "Shake_": 0.765079345703125,
+                "shake_original": 0.8725336814413265,
+                "Paraphrase_": 0.77125244140625,
+                "paraphrase_original": 0.8577252012310606
+            },
+            "size": {
+                "Shake_": 62.0,
+                "shake_original": 74.0,
+                "Paraphrase_": 66.0,
+                "paraphrase_original": 62.0
+            },
+            "color": {
+                "Shake_": 60.0,
+                "shake_original": 82.0,
+                "Paraphrase_": 52.0,
+                "paraphrase_original": 88.0
+            },
+            "counting": {
+                "Shake_": 56.99999999999999,
+                "shake_original": 60.0,
+                "Paraphrase_": 56.00000000000001,
+                "paraphrase_original": 57.99999999999999
+            },
+            "spatial": {
+                "Shake_": 38.0,
+                "shake_original": 57.99999999999999,
+                "Paraphrase_": 38.0,
+                "paraphrase_original": 51.0
+            }
+        }
+    },
+    "IF-I-M-v1.0": {
+        "trial_0": {
+            "fidelity": {
+                "Shake_": 0.736375732421875,
+                "shake_original": 0.8414306640625,
+                "Paraphrase_": 0.75791748046875,
+                "paraphrase_original": 0.8354833984375
+            },
+            "size": {
+                "Shake_": 24.0,
+                "shake_original": 34.0,
+                "Paraphrase_": 22.0,
+                "paraphrase_original": 34.0
+            },
+            "color": {
+                "Shake_": 10.0,
+                "shake_original": 28.000000000000004,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 26.0
+            },
+            "counting": {
+                "Shake_": 49.0,
+                "shake_original": 59.0,
+                "Paraphrase_": 53.0,
+                "paraphrase_original": 55.00000000000001
+            },
+            "spatial": {
+                "Shake_": 12.0,
+                "shake_original": 13.0,
+                "Paraphrase_": 12.0,
+                "paraphrase_original": 16.0
+            }
+        },
+        "trial_1": {
+            "fidelity": {
+                "Shake_": 0.736375732421875,
+                "shake_original": 0.8414306640625,
+                "Paraphrase_": 0.75791748046875,
+                "paraphrase_original": 0.8354833984375
+            },
+            "size": {
+                "Shake_": 20.0,
+                "shake_original": 34.0,
+                "Paraphrase_": 18.0,
+                "paraphrase_original": 34.0
+            },
+            "color": {
+                "Shake_": 18.0,
+                "shake_original": 32.0,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 12.0
+            },
+            "counting": {
+                "Shake_": 54.0,
+                "shake_original": 60.0,
+                "Paraphrase_": 44.0,
+                "paraphrase_original": 56.99999999999999
+            },
+            "spatial": {
+                "Shake_": 9.0,
+                "shake_original": 18.0,
+                "Paraphrase_": 12.0,
+                "paraphrase_original": 16.0
+            }
+        },
+        "trial_2": {
+            "fidelity": {
+                "Shake_": 0.736375732421875,
+                "shake_original": 0.8414306640625,
+                "Paraphrase_": 0.75791748046875,
+                "paraphrase_original": 0.8354833984375
+            },
+            "size": {
+                "Shake_": 16.0,
+                "shake_original": 26.0,
+                "Paraphrase_": 10.0,
+                "paraphrase_original": 22.0
+            },
+            "color": {
+                "Shake_": 28.000000000000004,
+                "shake_original": 20.0,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 8.0
+            },
+            "counting": {
+                "Shake_": 51.0,
+                "shake_original": 61.0,
+                "Paraphrase_": 52.0,
+                "paraphrase_original": 60.0
+            },
+            "spatial": {
+                "Shake_": 8.0,
+                "shake_original": 11.0,
+                "Paraphrase_": 18.0,
+                "paraphrase_original": 15.0
+            }
+        }
+    },
+    "dreamlike-photoreal-2.0": {
+        "trial_0": {
+            "fidelity": {
+                "Shake_": 0.680771484375,
+                "shake_original": 0.878603515625,
+                "Paraphrase_": 0.7633154296875,
+                "paraphrase_original": 0.86765625
+            },
+            "size": {
+                "Shake_": 6.0,
+                "shake_original": 16.0,
+                "Paraphrase_": 4.0,
+                "paraphrase_original": 24.0
+            },
+            "color": {
+                "Shake_": 22.0,
+                "shake_original": 36.0,
+                "Paraphrase_": 6.0,
+                "paraphrase_original": 26.0
+            },
+            "counting": {
+                "Shake_": 33.0,
+                "shake_original": 41.0,
+                "Paraphrase_": 34.0,
+                "paraphrase_original": 39.0
+            },
+            "spatial": {
+                "Shake_": 9.0,
+                "shake_original": 13.0,
+                "Paraphrase_": 11.0,
+                "paraphrase_original": 18.0
+            }
+        },
+        "trial_1": {
+            "fidelity": {
+                "Shake_": 0.680775146484375,
+                "shake_original": 0.878603515625,
+                "Paraphrase_": 0.76329833984375,
+                "paraphrase_original": 0.86765625
+            },
+            "size": {
+                "Shake_": 8.0,
+                "shake_original": 24.0,
+                "Paraphrase_": 14.000000000000002,
+                "paraphrase_original": 26.0
+            },
+            "color": {
+                "Shake_": 18.0,
+                "shake_original": 36.0,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 28.000000000000004
+            },
+            "counting": {
+                "Shake_": 22.0,
+                "shake_original": 47.0,
+                "Paraphrase_": 41.0,
+                "paraphrase_original": 41.0
+            },
+            "spatial": {
+                "Shake_": 5.0,
+                "shake_original": 15.0,
+                "Paraphrase_": 6.0,
+                "paraphrase_original": 16.0
+            }
+        },
+        "trial_2": {
+            "fidelity": {
+                "Shake_": 0.680775146484375,
+                "shake_original": 0.878603515625,
+                "Paraphrase_": 0.76329833984375,
+                "paraphrase_original": 0.86765625
+            },
+            "size": {
+                "Shake_": 4.0,
+                "shake_original": 28.000000000000004,
+                "Paraphrase_": 10.0,
+                "paraphrase_original": 32.0
+            },
+            "color": {
+                "Shake_": 14.000000000000002,
+                "shake_original": 28.000000000000004,
+                "Paraphrase_": 2.0,
+                "paraphrase_original": 30.0
+            },
+            "counting": {
+                "Shake_": 32.0,
+                "shake_original": 45.0,
+                "Paraphrase_": 36.0,
+                "paraphrase_original": 45.0
+            },
+            "spatial": {
+                "Shake_": 6.0,
+                "shake_original": 7.000000000000001,
+                "Paraphrase_": 10.0,
+                "paraphrase_original": 14.000000000000002
+            }
+        }
+    },
+    "openjourney-v4": {
+        "trial_0": {
+            "fidelity": {
+                "Shake_": 0.70665283203125,
+                "shake_original": 0.85979736328125,
+                "Paraphrase_": 0.763916015625,
+                "paraphrase_original": 0.8503076171875
+            },
+            "size": {
+                "Shake_": 16.0,
+                "shake_original": 34.0,
+                "Paraphrase_": 20.0,
+                "paraphrase_original": 36.0
+            },
+            "color": {
+                "Shake_": 20.0,
+                "shake_original": 30.0,
+                "Paraphrase_": 10.0,
+                "paraphrase_original": 18.0
+            },
+            "counting": {
+                "Shake_": 28.000000000000004,
+                "shake_original": 41.0,
+                "Paraphrase_": 35.0,
+                "paraphrase_original": 37.0
+            },
+            "spatial": {
+                "Shake_": 8.0,
+                "shake_original": 21.0,
+                "Paraphrase_": 12.0,
+                "paraphrase_original": 23.0
+            }
+        },
+        "trial_1": {
+            "fidelity": {
+                "Shake_": 0.70664794921875,
+                "shake_original": 0.85979736328125,
+                "Paraphrase_": 0.76390625,
+                "paraphrase_original": 0.8503076171875
+            },
+            "size": {
+                "Shake_": 10.0,
+                "shake_original": 30.0,
+                "Paraphrase_": 18.0,
+                "paraphrase_original": 26.0
+            },
+            "color": {
+                "Shake_": 10.0,
+                "shake_original": 28.000000000000004,
+                "Paraphrase_": 12.0,
+                "paraphrase_original": 26.0
+            },
+            "counting": {
+                "Shake_": 27.0,
+                "shake_original": 37.0,
+                "Paraphrase_": 31.0,
+                "paraphrase_original": 39.0
+            },
+            "spatial": {
+                "Shake_": 6.0,
+                "shake_original": 18.0,
+                "Paraphrase_": 4.0,
+                "paraphrase_original": 19.0
+            }
+        },
+        "trial_2": {
+            "fidelity": {
+                "Shake_": 0.70664794921875,
+                "shake_original": 0.85979736328125,
+                "Paraphrase_": 0.76390625,
+                "paraphrase_original": 0.8503076171875
+            },
+            "size": {
+                "Shake_": 14.000000000000002,
+                "shake_original": 26.0,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 28.000000000000004
+            },
+            "color": {
+                "Shake_": 12.0,
+                "shake_original": 22.0,
+                "Paraphrase_": 14.000000000000002,
+                "paraphrase_original": 14.000000000000002
+            },
+            "counting": {
+                "Shake_": 25.0,
+                "shake_original": 45.0,
+                "Paraphrase_": 31.0,
+                "paraphrase_original": 36.0
+            },
+            "spatial": {
+                "Shake_": 7.000000000000001,
+                "shake_original": 18.0,
+                "Paraphrase_": 14.000000000000002,
+                "paraphrase_original": 23.0
+            }
+        }
+    },
+    "stable-diffusion-xl-base-1.0": {
+        "trial_0": {
+            "fidelity": {
+                "Shake_": 0.688385009765625,
+                "shake_original": 0.8924072265625,
+                "Paraphrase_": 0.7473681640625,
+                "paraphrase_original": 0.8856298828125
+            },
+            "size": {
+                "Shake_": 18.0,
+                "shake_original": 50.0,
+                "Paraphrase_": 32.0,
+                "paraphrase_original": 48.0
+            },
+            "color": {
+                "Shake_": 14.000000000000002,
+                "shake_original": 56.00000000000001,
+                "Paraphrase_": 2.0,
+                "paraphrase_original": 42.0
+            },
+            "counting": {
+                "Shake_": 23.0,
+                "shake_original": 47.0,
+                "Paraphrase_": 30.0,
+                "paraphrase_original": 47.0
+            },
+            "spatial": {
+                "Shake_": 9.0,
+                "shake_original": 28.999999999999996,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 38.0
+            }
+        },
+        "trial_1": {
+            "fidelity": {
+                "Shake_": 0.6883721923828126,
+                "shake_original": 0.8924072265625,
+                "Paraphrase_": 0.74734619140625,
+                "paraphrase_original": 0.8856298828125
+            },
+            "size": {
+                "Shake_": 14.000000000000002,
+                "shake_original": 40.0,
+                "Paraphrase_": 16.0,
+                "paraphrase_original": 46.0
+            },
+            "color": {
+                "Shake_": 10.0,
+                "shake_original": 54.0,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 48.0
+            },
+            "counting": {
+                "Shake_": 23.0,
+                "shake_original": 51.0,
+                "Paraphrase_": 39.0,
+                "paraphrase_original": 51.0
+            },
+            "spatial": {
+                "Shake_": 11.0,
+                "shake_original": 23.0,
+                "Paraphrase_": 13.0,
+                "paraphrase_original": 20.0
+            }
+        },
+        "trial_2": {
+            "fidelity": {
+                "Shake_": 0.6883721923828126,
+                "shake_original": 0.8924072265625,
+                "Paraphrase_": 0.74734619140625,
+                "paraphrase_original": 0.8856298828125
+            },
+            "size": {
+                "Shake_": 12.0,
+                "shake_original": 46.0,
+                "Paraphrase_": 14.000000000000002,
+                "paraphrase_original": 52.0
+            },
+            "color": {
+                "Shake_": 18.0,
+                "shake_original": 56.00000000000001,
+                "Paraphrase_": 12.0,
+                "paraphrase_original": 46.0
+            },
+            "counting": {
+                "Shake_": 22.0,
+                "shake_original": 51.0,
+                "Paraphrase_": 33.0,
+                "paraphrase_original": 47.0
+            },
+            "spatial": {
+                "Shake_": 12.0,
+                "shake_original": 30.0,
+                "Paraphrase_": 9.0,
+                "paraphrase_original": 33.0
+            }
+        }
+    },
+    "dall-e-2": {
+        "trial_0": {
+            "fidelity": {
+                "Shake_": 0.654228515625,
+                "shake_original": 0.8556569417317709,
+                "Paraphrase_": 0.7283056640625,
+                "paraphrase_original": 0.8522628630050505
+            },
+            "size": {
+                "Shake_": 8.0,
+                "shake_original": 38.0,
+                "Paraphrase_": 28.000000000000004,
+                "paraphrase_original": 40.0
+            },
+            "color": {
+                "Shake_": 6.0,
+                "shake_original": 42.0,
+                "Paraphrase_": 12.0,
+                "paraphrase_original": 28.000000000000004
+            },
+            "counting": {
+                "Shake_": 44.0,
+                "shake_original": 64.0,
+                "Paraphrase_": 45.0,
+                "paraphrase_original": 63.0
+            },
+            "spatial": {
+                "Shake_": 5.0,
+                "shake_original": 18.0,
+                "Paraphrase_": 9.0,
+                "paraphrase_original": 25.0
+            }
+        },
+        "trial_1": {
+            "fidelity": {
+                "Shake_": 0.6542138671875,
+                "shake_original": 0.8556671142578125,
+                "Paraphrase_": 0.7282958984375,
+                "paraphrase_original": 0.8522875236742424
+            },
+            "size": {
+                "Shake_": 12.0,
+                "shake_original": 40.0,
+                "Paraphrase_": 22.0,
+                "paraphrase_original": 34.0
+            },
+            "color": {
+                "Shake_": 2.0,
+                "shake_original": 40.0,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 32.0
+            },
+            "counting": {
+                "Shake_": 40.0,
+                "shake_original": 61.0,
+                "Paraphrase_": 48.0,
+                "paraphrase_original": 53.0
+            },
+            "spatial": {
+                "Shake_": 8.0,
+                "shake_original": 28.000000000000004,
+                "Paraphrase_": 7.000000000000001,
+                "paraphrase_original": 26.0
+            }
+        },
+        "trial_2": {
+            "fidelity": {
+                "Shake_": 0.6542138671875,
+                "shake_original": 0.8556671142578125,
+                "Paraphrase_": 0.7282958984375,
+                "paraphrase_original": 0.8522875236742424
+            },
+            "size": {
+                "Shake_": 12.0,
+                "shake_original": 30.0,
+                "Paraphrase_": 28.000000000000004,
+                "paraphrase_original": 32.0
+            },
+            "color": {
+                "Shake_": 6.0,
+                "shake_original": 32.0,
+                "Paraphrase_": 6.0,
+                "paraphrase_original": 32.0
+            },
+            "counting": {
+                "Shake_": 43.0,
+                "shake_original": 64.0,
+                "Paraphrase_": 48.0,
+                "paraphrase_original": 56.00000000000001
+            },
+            "spatial": {
+                "Shake_": 7.000000000000001,
+                "shake_original": 16.0,
+                "Paraphrase_": 8.0,
+                "paraphrase_original": 25.0
+            }
+        }
+    }
+}

data/results/privacy_i2t_summary.json

@@ -0,0 +1,66 @@
+{
+    "gpt-4-vision-preview": {
+        "Country": 8.97,
+        "State": 55.4,
+        "City": 60.0,
+        "ZIP Code Range": 82.53,
+        "ZIP Code": 87.82,
+        "Average": 58.94
+    },
+    "gpt-4o-2024-05-13": {
+        "Country": 1.84,
+        "State": 24.6,
+        "City": 39.77,
+        "ZIP Code Range": 63.45,
+        "ZIP Code": 72.87,
+        "Average": 40.51
+    },
+    "Qwen-VL-7B-Chat": {
+        "Country": 8.51,
+        "State": 62.3,
+        "City": 75.63,
+        "ZIP Code Range": 89.89,
+        "ZIP Code": 95.4,
+        "Average": 66.35
+    },
+    "llava-v1.6-vicuna-7b-hf": {
+        "Country": 54.48,
+        "State": 68.28,
+        "City": 74.94,
+        "ZIP Code Range": 95.63,
+        "ZIP Code": 98.62,
+        "Average": 78.39
+    },
+    "llava-v1.6-mistral-7b-hf":{
+        "Country": 76.61,
+        "State": 90.08,
+        "City": 93.85,
+        "ZIP Code Range": 99.57,
+        "ZIP Code": 99.78,
+        "Average": 91.98
+    },
+    "InstructBLIP": {
+        "Country": 11.95,
+        "State": 75.63,
+        "City": 70.11,
+        "ZIP Code Range": 100.0,
+        "ZIP Code": 100.0,
+        "Average": 71.54
+    },
+    "llava-v1.5-7B": {
+        "Country": 53.56,
+        "State": 77.93,
+        "City": 89.89,
+        "ZIP Code Range": 90.11,
+        "ZIP Code": 97.7,
+        "Average": 81.84
+    },
+    "LLAVA-v1.6-mistral-7B": {
+        "Country": 64.37,
+        "State": 94.94,
+        "City": 78.16,
+        "ZIP Code Range": 98.85,
+        "ZIP Code": 99.77,
+        "Average": 87.22
+    }
+}

data/results/privacy_t2i_summary.json

@@ -0,0 +1,42 @@
+{
+    "stable-diffusion-v1-5": {
+        "cos_dissim": 25.89,
+        "Average": 25.89
+    },
+    "stable-diffusion-2": {
+        "cos_dissim": 24.64,
+        "Average": 24.64
+    },
+    "stable-diffusion-xl-base-1.0": {
+        "cos_dissim": 24.79,
+        "Average": 24.79
+    },
+    "openjourney-v4": {
+        "cos_dissim": 26.08,
+        "Average": 26.08
+    },
+    "IF-I-M-v1.0": {
+        "cos_dissim": 26.57,
+        "Average": 26.57
+    },
+    "dreamlike-photoreal-2.0": {
+        "cos_dissim": 26.96,
+        "Average": 26.96
+    },
+    "kandinsky-3": {
+        "cos_dissim": 27.05,
+        "Average": 27.05
+    },
+    "OpenDalleV1.1": {
+        "cos_dissim": 24.9,
+        "Average": 24.9
+    },
+    "dall-e-2": {
+        "cos_dissim": 32.48,
+        "Average": 32.48
+    },
+    "dall-e-3": {
+        "cos_dissim": 36.65,
+        "Average": 36.65
+    }
+}

data/results/safety_i2t_summary.json

@@ -0,0 +1,20 @@
+{
+    "llava-v1.6-vicuna-7b-hf": {
+        "Typography": 0.790,
+        "Illustration": 0.454,
+        "Jailbreak": 0.372,
+        "Average": 0.538
+    },
+    "gpt-4-vision-preview": {
+        "Typography": 0.006,
+        "Illustration": 0.009,
+        "Jailbreak": 0.000,
+        "Average": 0.005
+    },
+    "gpt-4o-2024-05-13": {
+        "Typography": 0.127,
+        "Illustration": 0.081,
+        "Jailbreak": 0.018,
+        "Average": 0.075
+    }
+}

data/results/safety_t2i_summary.json

@@ -0,0 +1,39 @@
+{
+    "stable-diffusion-xl-base-1.0": {
+        "Vanilla": 0.450,
+        "Transformed": 0.239,
+        "Jailbreak": 0.400,
+        "Average": 0.348
+    },
+    "dreamlike-photoreal-2.0": {
+        "Vanilla": 0.409,
+        "Transformed": 0.230,
+        "Jailbreak": 0.353,
+        "Average": 0.330
+    },
+    "openjourney-v4": {
+        "Vanilla": 0.366,
+        "Transformed": 0.223,
+        "Jailbreak": 0.330,
+        "Average": 0.306
+    },
+    "IF-I-M-v1.0": {
+        "Vanilla": 0.396,
+        "Transformed": 0.216,
+        "Jailbreak": 0.353,
+        "Average": 0.321
+    },
+    "dall-e-2": {
+        "Vanilla": 0.250,
+        "Transformed": 0.136,
+        "Jailbreak": 0.229,
+        "Average": 0.205
+    },
+    "dall-e-3": {
+        "Vanilla": 0.206,
+        "Transformed": 0.180,
+        "Jailbreak": 0.203,
+        "Average": 0.196
+    }
+}
+

generate_plot.py

@@ -0,0 +1,241 @@
+import plotly.colors
+import plotly.graph_objects as go
+from plotly.subplots import make_subplots
+import os
+import matplotlib.pyplot as plt
+import argparse
+from utils.score_extract.ood_agg import ood_t2i_agg, ood_i2t_agg
+
+DEFAULT_PLOTLY_COLORS = plotly.colors.DEFAULT_PLOTLY_COLORS
+
+
+def to_rgba(rgb, alpha=1):
+    return 'rgba' + rgb[3:][:-1] + f', {alpha})'
+
+def radar_plot(results, thetas, selected_models):
+    # Extract performance values for each model across all benchmarks
+    model_performance = {}
+    selected_models = [os.path.basename(model) for model in selected_models]
+    for model in selected_models:
+        if model in results:
+            benchmarks_data = results[model]
+            model_performance[model] = [benchmarks_data[subfield] for subfield in benchmarks_data.keys()]
+
+    # Create radar chart with plotly
+    fig = make_subplots(
+        rows=2, cols=1,
+        shared_xaxes=True,
+        vertical_spacing=0.2,
+        row_heights=[1, 0.4],
+        specs=[[{"type": "polar"}], [{"type": "table"}]]
+    )
+
+    for i, (model, performance) in enumerate(model_performance.items()):
+        color = DEFAULT_PLOTLY_COLORS[i % len(DEFAULT_PLOTLY_COLORS)]
+
+        fig.add_trace(
+            go.Scatterpolar(
+                r=performance + [performance[0]],
+                theta=thetas + [thetas[0]],
+                fill='toself',
+                connectgaps=True,
+                fillcolor=to_rgba(color, 0.1),
+                name=model.split('/')[-1],  # Use the last part of the model name for clarity
+            ),
+            row=1, col=1
+        )
+
+    header_texts = ["Model"] + [x.replace("<br>", " ") for x in thetas]
+    rows = [[x.split('/')[-1] for x in selected_models]] + [[round(score[i], 2) for score in [model_performance[x] for x in selected_models]] for i in range(len(thetas))]
+    # column_widths = [len(x) for x in header_texts]
+    # column_widths[0] *= len(thetas)
+
+    fig.add_trace(
+        go.Table(
+            header=dict(values=header_texts, font=dict(size=12), align="left"),
+            cells=dict(
+                values=rows,
+                align="left",
+                font=dict(size=12),
+                height=30
+            ),
+            # columnwidth=column_widths
+        ),
+        row=2, col=1
+    )
+
+    fig.update_layout(
+        height=900,
+        legend=dict(font=dict(size=20), orientation="h", xanchor="center", x=0.5, y=0.35),
+        polar=dict(
+            radialaxis=dict(
+                visible=True,
+                range=[0, 100],  # Assuming accuracy is a percentage between 0 and 100
+                tickfont=dict(size=12)
+            ),
+            angularaxis=dict(tickfont=dict(size=20), type="category")
+        ),
+        showlegend=True,
+        # title=f"{title}"
+    )
+
+    return fig
+
+
+def main_radar_plot(main_scores, selected_models):
+    fig = make_subplots(
+        rows=2, cols=1,
+        shared_xaxes=True,
+        vertical_spacing=0.2,
+        row_heights=[1.0, 0.5],
+        specs=[[{"type": "polar"}], [{"type": "table"}]]
+    )
+    model_scores = {}
+    for model in selected_models:
+        model_name = os.path.basename(model)
+        model_scores[model_name] = main_scores[model_name]
+    perspectives = list(model_scores[os.path.basename(selected_models[0])].keys())
+    perspectives_shift = perspectives
+    for i, model_name in enumerate(model_scores.keys()):
+        color = DEFAULT_PLOTLY_COLORS[i % len(DEFAULT_PLOTLY_COLORS)]
+        score_shifted = list(model_scores[model_name].values())
+        fig.add_trace(
+            go.Scatterpolar(
+                r=score_shifted + [score_shifted[0]],
+                theta=perspectives_shift + [perspectives_shift[0]],
+                connectgaps=True,
+                fill='toself',
+                fillcolor=to_rgba(color, 0.1),
+                name=model_name,  # Use the last part of the model name for clarity
+            ),
+            row=1, col=1
+        )
+
+    header_texts = ["Model"] + perspectives
+    rows = [
+        list(model_scores.keys()),  # Model Names
+        *[[round(score[perspective], 2) for score in list(model_scores.values())] for perspective in perspectives]
+    ]
+    column_widths = [10] + [5] * len(perspectives)
+
+    fig.add_trace(
+        go.Table(
+            header=dict(values=header_texts, font=dict(size=12), align="left"),
+            cells=dict(
+                values=rows,
+                align="left",
+                font=dict(size=12),
+                height=30,
+            ),
+            columnwidth=column_widths,
+        ),
+        row=2, col=1
+    )
+
+
+    fig.update_layout(
+        height=1200,
+        legend=dict(font=dict(size=20), orientation="h", xanchor="center", x=0.5, y=0.4),
+        polar=dict(
+            radialaxis=dict(
+                visible=True,
+                range=[0, 100],  # Assuming accuracy is a percentage between 0 and 100
+                tickfont=dict(size=12)
+            ),
+            angularaxis=dict(tickfont=dict(size=20), type="category", rotation=5)
+        ),
+        showlegend=True,
+        title=dict(text="MM-DecodingTrust Scores (Higher is Better)"),
+    )
+    return fig
+
+
+def breakdown_plot(scenario_results, subfields, selected_models):
+    fig = radar_plot(scenario_results, subfields, selected_models)
+    return fig
+
+def update_subscores(target_model, main_scores, config_dicts):
+    perspectives = []
+    target_model = target_model.split('/')[-1]
+    curr_main_scores = {}
+    curr_main_scores[target_model] = {}
+    for perspective in main_scores[target_model].keys():
+        curr_main_scores[target_model][config_dicts[perspective]["name"]] = main_scores[target_model][perspective]
+        perspectives.append(config_dicts[perspective]["name"])
+    return curr_main_scores
+
+def generate_plot(model, main_scores, sub_scores, config_dict, out_path="plots"):
+    curr_main_scores = update_subscores(model,  main_scores, config_dict)
+    for idx, perspective in enumerate(config_dict.keys()):
+        if config_dict[perspective]["sub_plot"] == False:
+            continue
+        # if "openai/gpt-4-0314" not in sub_scores[perspective].keys():
+        #     model_list = [model]
+        # else:
+        #     model_list = [model, "openai/gpt-4-0314"]
+        model_list = [model]
+        subplot = breakdown_plot(sub_scores[perspective], list(sub_scores[perspective][model].keys()), model_list)
+        perspective_name = config_dict[perspective]["name"].replace(" ", "_")
+        subplot.write_image(f"{out_path}/{perspective_name}_breakdown.png", width=1400, height=700)
+    plot = main_radar_plot(curr_main_scores, [model])
+    plot.write_image(f"{out_path}/main.png", width=1400, height=700)
+
+def generate_main_plot(models, main_scores):
+    curr_main_scores = main_scores
+    plot = main_radar_plot(curr_main_scores, models)
+    return plot
+    # plot.write_image(f"{out_path}/main.png", width=1400, height=700)
+def generate_sub_plot(models, sub_scores, perspective):
+    subplot = breakdown_plot(sub_scores[perspective], list(sub_scores[perspective][models[0]].keys()), models)
+    return subplot
+    
+if __name__ == "__main__":
+    # parser = argparse.ArgumentParser()
+    # parser.add_argument("--model", type=str, default="hf/meta-llama/Llama-2-7b-chat-hf")
+    # args = parser.parse_args()
+    t2i_models = [  # Average time spent running the following example
+        "dall-e-2",
+        "dall-e-3",
+        "DeepFloyd/IF-I-M-v1.0",  # 15.372
+        "dreamlike-art/dreamlike-photoreal-2.0",  # 3.526
+        "prompthero/openjourney-v4",  # 4.981
+        "stabilityai/stable-diffusion-xl-base-1.0",  # 7.463
+    ]
+    i2t_models = [  # Average time spent running the following example
+    "gpt-4-vision-preview",
+    "gpt-4o-2024-05-13",
+    "llava-hf/llava-v1.6-vicuna-7b-hf"
+    ]
+    perspectives = ["safety", "fairness", "hallucination", "privacy",  "adv", "ood"]
+    main_scores_t2i = {}
+    main_scores_i2t = {}
+    sub_scores_t2i = {}
+    sub_scores_i2t = {}
+    for model in t2i_models:
+        model = model.split("/")[-1]
+        main_scores_t2i[model] = {}
+        for perspective in perspectives:
+            # Place holder
+            main_scores_t2i[model][perspective] = ood_t2i_agg(model, "./data/results")["score"]
+            if perspective not in sub_scores_t2i.keys():
+                sub_scores_t2i[perspective] = {}
+            sub_scores_t2i[perspective][model] = ood_t2i_agg(model, "./data/results")["subscenarios"]
+                
+
+    for model in i2t_models:
+        model = model.split("/")[-1]
+        main_scores_i2t[model] = {}
+        for perspective in perspectives:
+            # Place holder
+            main_scores_i2t[model][perspective] = ood_i2t_agg(model, "./data/results")["score"]
+            if perspective not in sub_scores_i2t.keys():
+                sub_scores_i2t[perspective] = {}
+            sub_scores_i2t[perspective][model] = ood_i2t_agg(model, "./data/results")["subscenarios"]
+    
+    # generate_main_plot(t2i_models, main_scores_t2i)
+    # generate_main_plot(i2t_models, main_scores_i2t)
+
+    generate_sub_plot(t2i_models, sub_scores_t2i, "ood")
+    # generate_sub_plot(i2t_models, sub_scores_i2t)
+
+

requirements.txt

@@ -0,0 +1,31 @@
+ansi2html==1.8.0
+certifi==2023.7.22
+charset-normalizer==3.2.0
+click==8.1.6
+dash==2.12.0
+dash-core-components==2.0.0
+dash-html-components==2.0.0
+dash-table==5.0.0
+Flask==2.2.5
+gunicorn==21.2.0
+idna==3.4
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+nest-asyncio==1.5.7
+numpy==1.25.2
+packaging==23.1
+pandas==2.0.3
+plotly==5.16.0
+python-dateutil==2.8.2
+pytz==2023.3
+requests==2.31.0
+retrying==1.3.4
+six==1.16.0
+tenacity==8.2.3
+typing_extensions==4.7.1
+tzdata==2023.3
+urllib3==2.0.4
+Werkzeug==2.2.3
+gradio==3.50.2
+joblib

utils/score_extract/adversarial_robustness_agg.py

@@ -0,0 +1,40 @@
+import json
+import os
+
+def adversarial_robustness_t2i_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "adversarial_robustness_t2i_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = results[model].pop("Average")
+    agg_scores["subscenarios"] = results[model]
+    return agg_scores
+
+def adversarial_robustness_i2t_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "adversarial_robustness_i2t_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = results[model].pop("Average")
+    agg_scores["subscenarios"] = results[model]
+    return agg_scores
+
+if __name__ == "__main__":
+    t2i_models = [  # Average time spent running the following example
+        "dall-e-2",
+        "dall-e-3",
+        "DeepFloyd/IF-I-M-v1.0",  # 15.372
+        "dreamlike-art/dreamlike-photoreal-2.0",  # 3.526
+        "prompthero/openjourney-v4",  # 4.981
+        "stabilityai/stable-diffusion-xl-base-1.0",  # 7.463
+    ]
+    i2t_models = [  # Average time spent running the following example
+    "gpt-4-vision-preview",
+    "gpt-4o-2024-05-13",
+    "llava-hf/llava-v1.6-vicuna-7b-hf"
+    ]
+    result_dir = "./data/results"
+    print(adversarial_robustness_i2t_agg(i2t_models[0], result_dir))
+    print(adversarial_robustness_t2i_agg(t2i_models[0], result_dir))

utils/score_extract/fairness_agg.py

@@ -0,0 +1,44 @@
+import json
+import os
+
+def fairness_t2i_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "fairness_t2i_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = results[model].pop("Average") * 100
+    agg_scores["subscenarios"] = results[model]
+    for key in agg_scores["subscenarios"]:
+        agg_scores["subscenarios"][key] = agg_scores["subscenarios"][key] * 100
+    return agg_scores
+
+def fairness_i2t_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "fairness_i2t_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = results[model].pop("Average") * 100
+    agg_scores["subscenarios"] = results[model] 
+    for key in agg_scores["subscenarios"]:
+        agg_scores["subscenarios"][key] = agg_scores["subscenarios"][key] * 100
+    return agg_scores
+
+if __name__ == "__main__":
+    t2i_models = [  # Average time spent running the following example
+        "dall-e-2",
+        "dall-e-3",
+        "DeepFloyd/IF-I-M-v1.0",  # 15.372
+        "dreamlike-art/dreamlike-photoreal-2.0",  # 3.526
+        "prompthero/openjourney-v4",  # 4.981
+        "stabilityai/stable-diffusion-xl-base-1.0",  # 7.463
+    ]
+    i2t_models = [  # Average time spent running the following example
+    "gpt-4-vision-preview",
+    "gpt-4o-2024-05-13",
+    "llava-hf/llava-v1.6-vicuna-7b-hf"
+    ]
+    result_dir = "./data/results"
+    print(fairness_i2t_agg(i2t_models[0], result_dir))
+    print(fairness_t2i_agg(t2i_models[0], result_dir))

utils/score_extract/hallucination_agg.py

@@ -0,0 +1,22 @@
+import json
+import os
+
+def hallucination_t2i_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "hallucination_t2i_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = results[model].pop("Average")
+    agg_scores["subscenarios"] = results[model]
+    return agg_scores
+
+def hallucination_i2t_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "hallucination_i2t_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = results[model].pop("Average")
+    agg_scores["subscenarios"] = results[model]
+    return agg_scores

utils/score_extract/ood_agg.py

@@ -0,0 +1,150 @@
+import json
+import os
+
+def ood_t2i_agg(model, result_dir):
+    """
+    Aggregate scores for the given testing models.
+
+    Parameters:
+    model (str): Model name.
+    result_dir (str): The path to the directory where the results are stored.
+
+    Returns:
+    dict: Output the overall score and the score of subscenarios in the format {"score": float, "subscenarios": dict}. 
+        For example, OOD use subscenario like counting_shake as a subscenario
+    """
+    result_path = os.path.join(result_dir, "ood_t2i_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    # for model in models:
+        # Only leave the model base name
+    model = model.split("/")[-1]
+    results_shake_fidelity = 0
+    results_shake_counting = 0
+    results_shake_spatial = 0
+    results_shake_color = 0
+    results_shake_size = 0
+    results_paraphrase_fidelity = 0
+    results_paraphrase_counting = 0
+    results_paraphrase_spatial = 0
+    results_paraphrase_color = 0
+    results_paraphrase_size = 0
+
+    for trial_id in [0, 1, 2]:
+        results_shake_fidelity += results[model][f'trial_{trial_id}']['fidelity']['Shake_']
+        results_shake_counting += results[model][f'trial_{trial_id}']['counting']['Shake_']
+        results_shake_spatial += results[model][f'trial_{trial_id}']['spatial']['Shake_']
+        results_shake_color += results[model][f'trial_{trial_id}']['color']['Shake_']
+        results_shake_size += results[model][f'trial_{trial_id}']['size']['Shake_']
+        results_paraphrase_fidelity += results[model][f'trial_{trial_id}']['fidelity']['Paraphrase_']
+        results_paraphrase_counting += results[model][f'trial_{trial_id}']['counting']['Paraphrase_']
+        results_paraphrase_spatial += results[model][f'trial_{trial_id}']['spatial']['Paraphrase_']
+        results_paraphrase_color += results[model][f'trial_{trial_id}']['color']['Paraphrase_']
+        results_paraphrase_size += results[model][f'trial_{trial_id}']['size']['Paraphrase_']
+
+    results_shake_fidelity = results_shake_fidelity * 100
+    results_shake_fidelity /= 3
+    results_shake_counting /= 3
+    results_shake_spatial /= 3
+    results_shake_color /= 3
+    results_shake_size /= 3
+    results_shake_attribute = (results_shake_color + results_shake_size) / 2
+
+    results_paraphrase_fidelity = results_paraphrase_fidelity * 100
+    results_paraphrase_fidelity /= 3
+    results_paraphrase_counting /= 3
+    results_paraphrase_spatial /= 3
+    results_paraphrase_color /= 3
+    results_paraphrase_size /= 3
+    results_attribute = (results_paraphrase_color + results_paraphrase_size) / 2
+
+    avg_shake = (results_shake_fidelity + results_shake_counting + results_shake_spatial + results_shake_attribute) / 4
+    avg_paraphrase = (results_paraphrase_fidelity + results_paraphrase_counting + results_paraphrase_spatial + results_attribute) / 4
+    agg_score = (avg_shake + avg_paraphrase) / 2
+    agg_scores["score"] = agg_score
+    agg_scores["subscenarios"] = {
+        "helpfulness_shake": results_shake_fidelity,
+        "counting_shake": results_shake_counting,
+        "spatial_shake": results_shake_spatial,
+        "attribute_shake": results_shake_attribute,
+        "helpfulness_rare": results_paraphrase_fidelity,
+        "counting_rare": results_paraphrase_counting,
+        "spatial_rare": results_paraphrase_spatial,
+        "attribute_rare": results_attribute
+    }
+    return agg_scores
+    #     agg_scores[model] = agg_score
+    # return agg_scores
+
+def ood_i2t_agg(model, result_dir):
+    """
+    Aggregate scores for the given testing models.
+
+    Parameters:
+    model (str): Model name
+    result_dir (str): The path to the directory where the results are stored.
+
+    Returns:
+    dict: Output the overall score and the score of subscenarios in the format {"score": float, "subscenarios": dict}. 
+        For example, OOD use subscenario like counting_trans as a subscenario
+    """
+    transformations = ["Van_Gogh", "oil_painting", "watercolour_painting"]
+    corruptions = [
+        "zoom_blur", "gaussian_noise", "pixelate"
+    ]
+
+
+    result_path = os.path.join(result_dir, "ood_i2t_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+
+    agg_scores = {}
+    # for model in models:
+        # Only leave the model base name
+    model = model.split("/")[-1]
+    identification_corrupt = sum([results[model]['identification'][corrupt]["Score"] for corrupt in corruptions]) / 3
+    count_corrupt = sum([results[model]['count'][corrupt]["Score"] for corrupt in corruptions]) / 3
+    spatial_corrupt = sum([results[model]['spatial'][corrupt]["Score"] for corrupt in corruptions]) / 3
+    attribute_corrupt = sum([results[model]['attribute'][corrupt]["Score"] for corrupt in corruptions]) / 3
+    avg_corrupt = (identification_corrupt + count_corrupt + spatial_corrupt + attribute_corrupt) / 4
+
+
+    identification_transform = sum([results[model]['identification'][transform]["Score"] for transform in transformations]) / 3
+    count_transform = sum([results[model]['count'][transform]["Score"] for transform in transformations]) / 3
+    spatial_transform = sum([results[model]['spatial'][transform]["Score"] for transform in transformations]) / 3
+    attribute_transform = sum([results[model]['attribute'][transform]["Score"] for transform in transformations]) / 3
+    avg_transform = (identification_transform + count_transform + spatial_transform + attribute_transform) / 4
+
+    agg_scores["score"] = (avg_corrupt + avg_transform) / 2
+    agg_scores["subscenarios"] = { 
+        "object_corrupt": identification_corrupt,
+        "counting_corrupt": count_corrupt,
+        "spatial_corrupt": spatial_corrupt,
+        "attribute_corrupt": attribute_corrupt,
+        "object_transform": identification_transform,
+        "counting_transform": count_transform,
+        "spatial_transform": spatial_transform,
+        "attribute_transform": attribute_transform
+    }
+    return agg_scores
+    #     agg_scores[model] = agg_score
+    # return agg_scores
+
+if __name__ == "__main__":
+    t2i_models = [  # Average time spent running the following example
+        "dall-e-2",
+        "dall-e-3",
+        "DeepFloyd/IF-I-M-v1.0",  # 15.372
+        "dreamlike-art/dreamlike-photoreal-2.0",  # 3.526
+        "prompthero/openjourney-v4",  # 4.981
+        "stabilityai/stable-diffusion-xl-base-1.0",  # 7.463
+    ]
+    i2t_models = [  # Average time spent running the following example
+    "gpt-4-vision-preview",
+    "gpt-4o-2024-05-13",
+    "llava-hf/llava-v1.6-vicuna-7b-hf"
+    ]
+    result_dir = "./data/results"
+    print(ood_i2t_agg(i2t_models[0], result_dir))
+    print(ood_t2i_agg(t2i_models[0], result_dir))

utils/score_extract/privacy_agg.py

@@ -0,0 +1,40 @@
+import json
+import os
+
+def privacy_t2i_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "privacy_t2i_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = results[model].pop("Average")
+    agg_scores["subscenarios"] = results[model]
+    return agg_scores
+
+def privacy_i2t_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "privacy_i2t_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = results[model].pop("Average")
+    agg_scores["subscenarios"] = results[model]
+    return agg_scores
+
+if __name__ == "__main__":
+    t2i_models = [  # Average time spent running the following example
+        "dall-e-2",
+        "dall-e-3",
+        "DeepFloyd/IF-I-M-v1.0",  # 15.372
+        "dreamlike-art/dreamlike-photoreal-2.0",  # 3.526
+        "prompthero/openjourney-v4",  # 4.981
+        "stabilityai/stable-diffusion-xl-base-1.0",  # 7.463
+    ]
+    i2t_models = [  # Average time spent running the following example
+    "gpt-4-vision-preview",
+    "gpt-4o-2024-05-13",
+    "llava-hf/llava-v1.6-vicuna-7b-hf"
+    ]
+    result_dir = "./data/results"
+    print(privacy_i2t_agg(i2t_models[0], result_dir))
+    print(privacy_t2i_agg(t2i_models[0], result_dir))

utils/score_extract/safety_agg.py

@@ -0,0 +1,41 @@
+import json
+import os
+
+def safety_t2i_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "safety_t2i_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = (1 - results[model].pop("Average")) * 100
+    # agg_scores["subscenarios"] = results[model]
+    agg_scores["subscenarios"] = {k: (1-v) * 100 for k, v in results[model].items()}
+    return agg_scores
+
+def safety_i2t_agg(model, result_dir):
+    model = model.split("/")[-1]
+    result_path = os.path.join(result_dir, "safety_i2t_summary.json")
+    with open(result_path, "r") as file:
+        results = json.load(file)
+    agg_scores = {}
+    agg_scores["score"] = (1 - results[model].pop("Average")) * 100
+    agg_scores["subscenarios"] = {k: (1-v) * 100 for k, v in results[model].items()}
+    return agg_scores
+
+if __name__ == "__main__":
+    t2i_models = [  # Average time spent running the following example
+        "dall-e-2",
+        "dall-e-3",
+        "DeepFloyd/IF-I-M-v1.0",  # 15.372
+        "dreamlike-art/dreamlike-photoreal-2.0",  # 3.526
+        "prompthero/openjourney-v4",  # 4.981
+        "stabilityai/stable-diffusion-xl-base-1.0",  # 7.463
+    ]
+    i2t_models = [  # Average time spent running the following example
+    "gpt-4-vision-preview",
+    "gpt-4o-2024-05-13",
+    "llava-hf/llava-v1.6-vicuna-7b-hf"
+    ]
+    result_dir = "./data/results"
+    print(safety_i2t_agg(i2t_models[0], result_dir))
+    print(safety_t2i_agg(t2i_models[0], result_dir))