Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/paligemma-3b-mix-448",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": false,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:961b72c2b2a1bebc3e11e7d98cd173d263437aad5ef36e760e9265c184c88d64
+size 78625112

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image>": 257152
+}

git_hash.txt

@@ -0,0 +1 @@
+83ba45daf034b63549ce091c9fc7d1b05d17381a

preprocessor_config.json

@@ -0,0 +1,40 @@
+{
+  "_valid_processor_keys": [
+    "images",
+    "do_resize",
+    "size",
+    "resample",
+    "do_rescale",
+    "rescale_factor",
+    "do_normalize",
+    "image_mean",
+    "image_std",
+    "return_tensors",
+    "data_format",
+    "input_data_format",
+    "do_convert_rgb"
+  ],
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "SiglipImageProcessor",
+  "image_seq_length": 1024,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "PaliGemmaProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 448,
+    "width": 448
+  }
+}

results.json

@@ -0,0 +1 @@
+{"validation_set": {"ndcg_at_1": 0.774, "ndcg_at_3": 0.83243, "ndcg_at_5": 0.84895, "ndcg_at_10": 0.85814, "ndcg_at_20": 0.8643, "ndcg_at_100": 0.86854, "ndcg_at_1000": 0.87018, "map_at_1": 0.774, "map_at_3": 0.81867, "map_at_5": 0.82787, "map_at_10": 0.83174, "map_at_20": 0.83348, "map_at_100": 0.83399, "map_at_1000": 0.83407, "recall_at_1": 0.774, "recall_at_3": 0.872, "recall_at_5": 0.912, "recall_at_10": 0.94, "recall_at_20": 0.964, "recall_at_100": 0.988, "recall_at_1000": 1.0, "precision_at_1": 0.774, "precision_at_3": 0.29067, "precision_at_5": 0.1824, "precision_at_10": 0.094, "precision_at_20": 0.0482, "precision_at_100": 0.00988, "precision_at_1000": 0.001, "mrr_at_1": 0.778, "mrr_at_3": 0.821, "mrr_at_5": 0.8286999999999999, "mrr_at_10": 0.8335484126984126, "mrr_at_20": 0.8347260450333979, "mrr_at_100": 0.8354386595905559, "mrr_at_1000": 0.8355249939420808, "naucs_at_1_max": -0.0145170587484219, "naucs_at_1_std": 0.006309262814839989, "naucs_at_1_diff1": 0.8950447836378629, "naucs_at_3_max": 0.021200161668434796, "naucs_at_3_std": 0.20331461011654423, "naucs_at_3_diff1": 0.8777295662755695, "naucs_at_5_max": 0.08481716987168389, "naucs_at_5_std": 0.35063874034462167, "naucs_at_5_diff1": 0.8777528653012975, "naucs_at_10_max": 0.09851459770146549, "naucs_at_10_std": 0.5862433862433853, "naucs_at_10_diff1": 0.9003930543110488, "naucs_at_20_max": -0.04060065920783268, "naucs_at_20_std": 0.6889718850503177, "naucs_at_20_diff1": 0.9353989134331586, "naucs_at_100_max": 0.4440696998264123, "naucs_at_100_std": 0.8537970743853125, "naucs_at_100_diff1": 0.9341234384048992, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": -0.0}, "syntheticDocQA_energy": {"ndcg_at_1": 0.85, "ndcg_at_3": 0.89786, "ndcg_at_5": 0.9099, "ndcg_at_10": 0.91346, "ndcg_at_20": 0.91346, "ndcg_at_100": 0.91879, "ndcg_at_1000": 0.91879, "map_at_1": 0.85, "map_at_3": 0.88667, "map_at_5": 0.89317, "map_at_10": 0.89483, "map_at_20": 0.89483, "map_at_100": 0.89545, "map_at_1000": 0.89545, "recall_at_1": 0.85, "recall_at_3": 0.93, "recall_at_5": 0.96, "recall_at_10": 0.97, "recall_at_20": 0.97, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.31, "precision_at_5": 0.192, "precision_at_10": 0.097, "precision_at_20": 0.0485, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.86, "mrr_at_3": 0.895, "mrr_at_5": 0.9009999999999999, "mrr_at_10": 0.9009999999999999, "mrr_at_20": 0.9009999999999999, "mrr_at_100": 0.9016280663780663, "mrr_at_1000": 0.9016280663780663, "naucs_at_1_max": 0.2105750182116147, "naucs_at_1_std": -0.1620891636836963, "naucs_at_1_diff1": 0.9412806884333085, "naucs_at_3_max": -0.038548439319115155, "naucs_at_3_std": -0.30092036814725776, "naucs_at_3_diff1": 0.9024795103226442, "naucs_at_5_max": 0.46231071243125327, "naucs_at_5_std": -0.20074696545284348, "naucs_at_5_diff1": 0.93100944081336, "naucs_at_10_max": 0.5739365102913813, "naucs_at_10_std": -0.45238095238095277, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.5739365102913813, "naucs_at_20_std": -0.45238095238095277, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_healthcare_industry": {"ndcg_at_1": 0.88, "ndcg_at_3": 0.93547, "ndcg_at_5": 0.94365, "ndcg_at_10": 0.94365, "ndcg_at_20": 0.94365, "ndcg_at_100": 0.94515, "ndcg_at_1000": 0.94515, "map_at_1": 0.88, "map_at_3": 0.92333, "map_at_5": 0.92783, "map_at_10": 0.92783, "map_at_20": 0.92783, "map_at_100": 0.92793, "map_at_1000": 0.92793, "recall_at_1": 0.88, "recall_at_3": 0.97, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.88, "precision_at_3": 0.32333, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.89, "mrr_at_3": 0.9283333333333332, "mrr_at_5": 0.9328333333333333, "mrr_at_10": 0.9328333333333333, "mrr_at_20": 0.9328333333333333, "mrr_at_100": 0.9329364261168385, "mrr_at_1000": 0.9329364261168385, "naucs_at_1_max": 0.3449858099592377, "naucs_at_1_std": -0.2542554883868911, "naucs_at_1_diff1": 0.8584721869131066, "naucs_at_3_max": 0.19373303220940802, "naucs_at_3_std": -1.0210084033613431, "naucs_at_3_diff1": 0.8314665575583057, "naucs_at_5_max": 0.3612909644732028, "naucs_at_5_std": -1.7399626517273863, "naucs_at_5_diff1": 0.8787143723468824, "naucs_at_10_max": 0.3612909644732028, "naucs_at_10_std": -1.7399626517273863, "naucs_at_10_diff1": 0.8787143723468824, "naucs_at_20_max": 0.3612909644732028, "naucs_at_20_std": -1.7399626517273863, "naucs_at_20_diff1": 0.8787143723468824, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.93, "ndcg_at_3": 0.96155, "ndcg_at_5": 0.96155, "ndcg_at_10": 0.96488, "ndcg_at_20": 0.96488, "ndcg_at_100": 0.96663, "ndcg_at_1000": 0.96663, "map_at_1": 0.93, "map_at_3": 0.955, "map_at_5": 0.955, "map_at_10": 0.95643, "map_at_20": 0.95643, "map_at_100": 0.95662, "map_at_1000": 0.95662, "recall_at_1": 0.93, "recall_at_3": 0.98, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.32667, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.93, "mrr_at_3": 0.955, "mrr_at_5": 0.955, "mrr_at_10": 0.9564285714285714, "mrr_at_20": 0.9564285714285714, "mrr_at_100": 0.9566285714285715, "mrr_at_1000": 0.9566285714285715, "naucs_at_1_max": -0.0004962491760935482, "naucs_at_1_std": -0.31752701080432255, "naucs_at_1_diff1": 0.8859352860974721, "naucs_at_3_max": -1.393329269612857, "naucs_at_3_std": -1.7399626517273692, "naucs_at_3_diff1": 0.9999999999999902, "naucs_at_5_max": -1.3933292696128265, "naucs_at_5_std": -1.7399626517273497, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": -1.1259495904684838, "naucs_at_10_std": -1.7399626517273863, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": -1.1259495904684838, "naucs_at_20_std": -1.7399626517273863, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_government_reports": {"ndcg_at_1": 0.85, "ndcg_at_3": 0.91417, "ndcg_at_5": 0.92709, "ndcg_at_10": 0.92998, "ndcg_at_20": 0.92998, "ndcg_at_100": 0.92998, "ndcg_at_1000": 0.92998, "map_at_1": 0.85, "map_at_3": 0.89833, "map_at_5": 0.90583, "map_at_10": 0.90683, "map_at_20": 0.90683, "map_at_100": 0.90683, "map_at_1000": 0.90683, "recall_at_1": 0.85, "recall_at_3": 0.96, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.32, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.86, "mrr_at_3": 0.9083333333333331, "mrr_at_5": 0.9133333333333331, "mrr_at_10": 0.9144444444444443, "mrr_at_20": 0.9144444444444443, "mrr_at_100": 0.9144444444444443, "mrr_at_1000": 0.9144444444444443, "naucs_at_1_max": 0.0728385886905621, "naucs_at_1_std": -0.27940123657663585, "naucs_at_1_diff1": 0.838404544865166, "naucs_at_3_max": 0.43264898785631317, "naucs_at_3_std": 0.3535247432306253, "naucs_at_3_diff1": 0.7950946429181002, "naucs_at_5_max": 1.0, "naucs_at_5_std": 0.8692810457516413, "naucs_at_5_diff1": 0.5715286132037668, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "infovqa_subsampled": {"ndcg_at_1": 0.746, "ndcg_at_3": 0.80343, "ndcg_at_5": 0.81832, "ndcg_at_10": 0.82857, "ndcg_at_20": 0.83312, "ndcg_at_100": 0.84258, "ndcg_at_1000": 0.84537, "map_at_1": 0.746, "map_at_3": 0.79, "map_at_5": 0.7983, "map_at_10": 0.80247, "map_at_20": 0.80372, "map_at_100": 0.8051, "map_at_1000": 0.8052, "recall_at_1": 0.746, "recall_at_3": 0.842, "recall_at_5": 0.878, "recall_at_10": 0.91, "recall_at_20": 0.928, "recall_at_100": 0.978, "recall_at_1000": 1.0, "precision_at_1": 0.746, "precision_at_3": 0.28067, "precision_at_5": 0.1756, "precision_at_10": 0.091, "precision_at_20": 0.0464, "precision_at_100": 0.00978, "precision_at_1000": 0.001, "mrr_at_1": 0.752, "mrr_at_3": 0.7953333333333332, "mrr_at_5": 0.8023333333333331, "mrr_at_10": 0.8065706349206349, "mrr_at_20": 0.8078986539174619, "mrr_at_100": 0.8091823645081221, "mrr_at_1000": 0.809263223979117, "naucs_at_1_max": 0.24674907189259504, "naucs_at_1_std": 0.11457666064661158, "naucs_at_1_diff1": 0.8740250282732915, "naucs_at_3_max": 0.23413123236282346, "naucs_at_3_std": 0.20991570348483488, "naucs_at_3_diff1": 0.8357975149102336, "naucs_at_5_max": 0.35286707023572256, "naucs_at_5_std": 0.37199604959947313, "naucs_at_5_diff1": 0.8207018895992207, "naucs_at_10_max": 0.3522596496009211, "naucs_at_10_std": 0.3307189542483652, "naucs_at_10_diff1": 0.8090976640617076, "naucs_at_20_max": 0.4744509713295461, "naucs_at_20_std": 0.47494553376906107, "naucs_at_20_diff1": 0.7939471631070897, "naucs_at_100_max": 0.44440793303781984, "naucs_at_100_std": 0.8327815974874812, "naucs_at_100_diff1": 0.8297168584345167, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "docvqa_subsampled": {"ndcg_at_1": 0.456, "ndcg_at_3": 0.52447, "ndcg_at_5": 0.54427, "ndcg_at_10": 0.56508, "ndcg_at_20": 0.58069, "ndcg_at_100": 0.60433, "ndcg_at_1000": 0.62051, "map_at_1": 0.456, "map_at_3": 0.508, "map_at_5": 0.519, "map_at_10": 0.52765, "map_at_20": 0.5319, "map_at_100": 0.53515, "map_at_1000": 0.53576, "recall_at_1": 0.456, "recall_at_3": 0.572, "recall_at_5": 0.62, "recall_at_10": 0.684, "recall_at_20": 0.746, "recall_at_100": 0.874, "recall_at_1000": 1.0, "precision_at_1": 0.456, "precision_at_3": 0.19067, "precision_at_5": 0.124, "precision_at_10": 0.0684, "precision_at_20": 0.0373, "precision_at_100": 0.00874, "precision_at_1000": 0.001, "mrr_at_1": 0.452, "mrr_at_3": 0.505, "mrr_at_5": 0.5164000000000001, "mrr_at_10": 0.5254579365079365, "mrr_at_20": 0.5290178752190362, "mrr_at_100": 0.5325203981148059, "mrr_at_1000": 0.5330986506794522, "naucs_at_1_max": 0.010704720337137848, "naucs_at_1_std": 0.4181494661921707, "naucs_at_1_diff1": 0.7688057111458236, "naucs_at_3_max": -0.0067852838115276684, "naucs_at_3_std": 0.408076792525558, "naucs_at_3_diff1": 0.7055654080347042, "naucs_at_5_max": -0.011161683506439484, "naucs_at_5_std": 0.452272097960188, "naucs_at_5_diff1": 0.6829497998035559, "naucs_at_10_max": -0.06663979903420814, "naucs_at_10_std": 0.4644934958160017, "naucs_at_10_diff1": 0.6670322281267268, "naucs_at_20_max": -0.08314881099503675, "naucs_at_20_std": 0.5636259190226841, "naucs_at_20_diff1": 0.6475212710859795, "naucs_at_100_max": -0.15012530637692442, "naucs_at_100_std": 0.8336557354576346, "naucs_at_100_diff1": 0.6215172139879154, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 2.0}, "arxivqa_subsampled": {"ndcg_at_1": 0.724, "ndcg_at_3": 0.77659, "ndcg_at_5": 0.7914, "ndcg_at_10": 0.80967, "ndcg_at_20": 0.81716, "ndcg_at_100": 0.82667, "ndcg_at_1000": 0.82854, "map_at_1": 0.724, "map_at_3": 0.76367, "map_at_5": 0.77187, "map_at_10": 0.77989, "map_at_20": 0.78189, "map_at_100": 0.78316, "map_at_1000": 0.78324, "recall_at_1": 0.724, "recall_at_3": 0.814, "recall_at_5": 0.85, "recall_at_10": 0.904, "recall_at_20": 0.934, "recall_at_100": 0.986, "recall_at_1000": 1.0, "precision_at_1": 0.724, "precision_at_3": 0.27133, "precision_at_5": 0.17, "precision_at_10": 0.0904, "precision_at_20": 0.0467, "precision_at_100": 0.00986, "precision_at_1000": 0.001, "mrr_at_1": 0.72, "mrr_at_3": 0.7606666666666666, "mrr_at_5": 0.7704666666666665, "mrr_at_10": 0.7783031746031746, "mrr_at_20": 0.7798739388062916, "mrr_at_100": 0.7811347665019601, "mrr_at_1000": 0.781219393810504, "naucs_at_1_max": 0.016575744452395444, "naucs_at_1_std": 0.14620481415983924, "naucs_at_1_diff1": 0.8319131181262124, "naucs_at_3_max": -0.025943897313719388, "naucs_at_3_std": 0.19927764658111108, "naucs_at_3_diff1": 0.7774553764676329, "naucs_at_5_max": -0.0448735069102902, "naucs_at_5_std": 0.26161405792385234, "naucs_at_5_diff1": 0.7677282039192751, "naucs_at_10_max": -0.10797954318975009, "naucs_at_10_std": 0.20837223778400116, "naucs_at_10_diff1": 0.7147052306317629, "naucs_at_20_max": -0.00911433048928522, "naucs_at_20_std": 0.4119910590498834, "naucs_at_20_diff1": 0.7000503290790163, "naucs_at_100_max": 0.5246753983500644, "naucs_at_100_std": 0.8289315726290336, "naucs_at_100_diff1": 0.6839271409113703, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "tabfquad_subsampled": {"ndcg_at_1": 0.75357, "ndcg_at_3": 0.82768, "ndcg_at_5": 0.83936, "ndcg_at_10": 0.85309, "ndcg_at_20": 0.85858, "ndcg_at_100": 0.86371, "ndcg_at_1000": 0.86371, "map_at_1": 0.75357, "map_at_3": 0.80893, "map_at_5": 0.81536, "map_at_10": 0.82095, "map_at_20": 0.8225, "map_at_100": 0.82339, "map_at_1000": 0.82339, "recall_at_1": 0.75357, "recall_at_3": 0.88214, "recall_at_5": 0.91071, "recall_at_10": 0.95357, "recall_at_20": 0.975, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.75357, "precision_at_3": 0.29405, "precision_at_5": 0.18214, "precision_at_10": 0.09536, "precision_at_20": 0.04875, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.7535714285714286, "mrr_at_3": 0.8101190476190477, "mrr_at_5": 0.8167261904761904, "mrr_at_10": 0.8221357709750566, "mrr_at_20": 0.8239959162569457, "mrr_at_100": 0.8248561140910201, "mrr_at_1000": 0.8248561140910201, "naucs_at_1_max": 0.5304566374894598, "naucs_at_1_std": 0.1615906461054432, "naucs_at_1_diff1": 0.8208688581619074, "naucs_at_3_max": 0.4836484318803355, "naucs_at_3_std": 0.21076407932338395, "naucs_at_3_diff1": 0.7733693395413521, "naucs_at_5_max": 0.4612358016225636, "naucs_at_5_std": 0.27449112978524876, "naucs_at_5_diff1": 0.7189851642886126, "naucs_at_10_max": 0.5759036686726754, "naucs_at_10_std": 0.32575594340300035, "naucs_at_10_diff1": 0.7358166602027056, "naucs_at_20_max": 0.47733910929211887, "naucs_at_20_std": 0.25596905428838695, "naucs_at_20_diff1": 0.7967511397858333, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": -0.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "tatdqa": {"ndcg_at_1": 0.53097, "ndcg_at_3": 0.62877, "ndcg_at_5": 0.65801, "ndcg_at_10": 0.68473, "ndcg_at_20": 0.69746, "ndcg_at_100": 0.71145, "ndcg_at_1000": 0.7146, "map_at_1": 0.53097, "map_at_3": 0.60443, "map_at_5": 0.62052, "map_at_10": 0.63158, "map_at_20": 0.63505, "map_at_100": 0.63713, "map_at_1000": 0.63728, "recall_at_1": 0.53097, "recall_at_3": 0.69934, "recall_at_5": 0.7709, "recall_at_10": 0.85328, "recall_at_20": 0.90379, "recall_at_100": 0.97715, "recall_at_1000": 1.0, "precision_at_1": 0.53097, "precision_at_3": 0.23311, "precision_at_5": 0.15418, "precision_at_10": 0.08533, "precision_at_20": 0.04519, "precision_at_100": 0.00977, "precision_at_1000": 0.001, "mrr_at_1": 0.5267588695129285, "mrr_at_3": 0.6032270996191631, "mrr_at_5": 0.61913209059932, "mrr_at_10": 0.6304894577594526, "mrr_at_20": 0.6337674299282028, "mrr_at_100": 0.6358829720309133, "mrr_at_1000": 0.6360412330135795, "naucs_at_1_max": 0.12207506980114474, "naucs_at_1_std": -0.06857703877057526, "naucs_at_1_diff1": 0.6350798605456752, "naucs_at_3_max": 0.13028071020881998, "naucs_at_3_std": -0.03204133285184471, "naucs_at_3_diff1": 0.536055692458893, "naucs_at_5_max": 0.14913490780176689, "naucs_at_5_std": 0.0019238783137197706, "naucs_at_5_diff1": 0.5114302291162272, "naucs_at_10_max": 0.152124608400107, "naucs_at_10_std": 0.10791232460205795, "naucs_at_10_diff1": 0.48383898061381275, "naucs_at_20_max": 0.1449130314981855, "naucs_at_20_std": 0.22594318462810645, "naucs_at_20_diff1": 0.43552881290118256, "naucs_at_100_max": -0.12684151303788002, "naucs_at_100_std": 0.5145680511335288, "naucs_at_100_diff1": 0.5095488271113825, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "shift_project": {"ndcg_at_1": 0.55, "ndcg_at_3": 0.71488, "ndcg_at_5": 0.73167, "ndcg_at_10": 0.74796, "ndcg_at_20": 0.75568, "ndcg_at_100": 0.76079, "ndcg_at_1000": 0.76366, "map_at_1": 0.55, "map_at_3": 0.675, "map_at_5": 0.6845, "map_at_10": 0.69131, "map_at_20": 0.6935, "map_at_100": 0.69406, "map_at_1000": 0.69422, "recall_at_1": 0.55, "recall_at_3": 0.83, "recall_at_5": 0.87, "recall_at_10": 0.92, "recall_at_20": 0.95, "recall_at_100": 0.98, "recall_at_1000": 1.0, "precision_at_1": 0.55, "precision_at_3": 0.27667, "precision_at_5": 0.174, "precision_at_10": 0.092, "precision_at_20": 0.0475, "precision_at_100": 0.0098, "precision_at_1000": 0.001, "mrr_at_1": 0.57, "mrr_at_3": 0.69, "mrr_at_5": 0.6970000000000001, "mrr_at_10": 0.7041230158730158, "mrr_at_20": 0.7063159983291563, "mrr_at_100": 0.7069244639376219, "mrr_at_1000": 0.7070891464773044, "naucs_at_1_max": -0.13381190246642352, "naucs_at_1_std": -0.30200098293898753, "naucs_at_1_diff1": 0.5594199548867977, "naucs_at_3_max": -0.2639588734077385, "naucs_at_3_std": -0.5422302200034823, "naucs_at_3_diff1": 0.6073825476848427, "naucs_at_5_max": -0.2204135305072195, "naucs_at_5_std": -0.4037250981263389, "naucs_at_5_diff1": 0.4991023586585799, "naucs_at_10_max": -0.16129215914007541, "naucs_at_10_std": -0.5218253968253913, "naucs_at_10_diff1": 0.4407238859736181, "naucs_at_20_max": -0.38175400241396046, "naucs_at_20_std": -0.7713352007469497, "naucs_at_20_diff1": 0.5734029421882291, "naucs_at_100_max": -1.1341349547949189, "naucs_at_100_std": -0.6909430438842147, "naucs_at_100_diff1": 0.48572411420708356, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}}

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1df2ab04780faccf51a881d7c5a7026cc6f979083af2eebf709d051b8d47134b
+size 17763458

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8986bb4f423f07f8c7f70d0dbe3526fb2316056c17bae71b1ea975e77a168fc6
+size 4264023

training_config.yml

@@ -0,0 +1,41 @@
+config:
+  (): custom_colbert.utils.train_custom_colbert_models.ColModelTrainingConfig
+  output_dir: !path ../../../models/without_tabfquad_no_pairwise/train_colpali-3b-mix-448
+  processor:
+    () : custom_colbert.utils.wrapper.AutoProcessorWrapper
+    pretrained_model_name_or_path: "./models/paligemma-3b-mix-448"
+    max_length: 50
+  model:
+    (): custom_colbert.utils.wrapper.AutoColModelWrapper
+    pretrained_model_name_or_path: "./models/paligemma-3b-mix-448"
+    training_objective: "colbertv1"
+    # attn_implementation: "eager"
+    torch_dtype:  !ext torch.bfloat16
+#    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext custom_colbert.utils.dataset_transformation.load_train_set
+  eval_dataset_loader: !import ../data/test_data.yaml
+
+  max_length: 50
+  run_eval: true
+  add_suffix: true
+  loss_func:
+    (): custom_colbert.loss.colbert_loss.ColbertLoss
+  tr_args: !import ../tr_args/default_tr_args.yaml
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+