Upload absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.8.hook_resid_post_eval_results.json with huggingface_hub

absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.8.hook_resid_post_eval_results.json

@@ -0,0 +1,275 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50,
+    "min_GT_probe_f1": 0.6,
+    "min_feats_for_eval": 20
+  },
+  "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6",
+  "datetime_epoch_millis": 1745719249324,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.00849608699429072,
+      "mean_full_absorption_score": 0.005332496852924044,
+      "mean_num_split_features": 1.1923076923076923,
+      "std_dev_absorption_fraction_score": 0.011257446352611816,
+      "std_dev_full_absorption_score": 0.0054153913868388364,
+      "std_dev_num_split_features": 0.4914656259988704
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.0015349194167306216,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 2606,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.0017258586584180155,
+      "full_absorption_rate": 0.007352941176470588,
+      "num_full_absorption": 12,
+      "num_probe_true_positives": 1632,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.0007065002407756726,
+      "full_absorption_rate": 0.0003619254433586681,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 2763,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.004667444574095682,
+      "num_full_absorption": 8,
+      "num_probe_true_positives": 1714,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.0012391573729863693,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 1614,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.022999228274488366,
+      "full_absorption_rate": 0.007518796992481203,
+      "num_full_absorption": 9,
+      "num_probe_true_positives": 1197,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.017162911537986707,
+      "full_absorption_rate": 0.005235602094240838,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 1146,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.00977314513127678,
+      "full_absorption_rate": 0.008695652173913044,
+      "num_full_absorption": 9,
+      "num_probe_true_positives": 1035,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.0013778418224468837,
+      "full_absorption_rate": 0.003486345148169669,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 1721,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.00856764628821801,
+      "full_absorption_rate": 0.00909090909090909,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 440,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.002424164950516951,
+      "full_absorption_rate": 0.001451378809869376,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 689,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.0001241204628941615,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 1220,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.004420601412823913,
+      "full_absorption_rate": 0.0011435105774728416,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 1749,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.0026002795862142694,
+      "full_absorption_rate": 0.0011148272017837235,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 897,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.016910753534243702,
+      "full_absorption_rate": 0.014299332697807437,
+      "num_full_absorption": 15,
+      "num_probe_true_positives": 1049,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.0008547008547008547,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 2340,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.007804842632591571,
+      "full_absorption_rate": 0.006134969325153374,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 163,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.029781651136315337,
+      "full_absorption_rate": 0.014226437462951986,
+      "num_full_absorption": 24,
+      "num_probe_true_positives": 1687,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.00312977151299912,
+      "full_absorption_rate": 0.0024946543121881683,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 2806,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.0010544198516813646,
+      "full_absorption_rate": 0.0006038647342995169,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 1656,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.043622843678462524,
+      "full_absorption_rate": 0.017834394904458598,
+      "num_full_absorption": 14,
+      "num_probe_true_positives": 785,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.0001906515352730492,
+      "full_absorption_rate": 0.0026560424966799467,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 753,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.0215796713199841,
+      "full_absorption_rate": 0.01422475106685633,
+      "num_full_absorption": 10,
+      "num_probe_true_positives": 703,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.0027988336693133666,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 109,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.019691286254293176,
+      "full_absorption_rate": 0.012422360248447204,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 161,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.002451238360341718,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 227,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "Unknown",
+  "sae_lens_id": "blocks.8.hook_resid_post",
+  "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc",
+  "sae_lens_version": "5.9.1",
+  "sae_cfg_dict": {
+    "architecture": "jumprelu",
+    "d_in": 2304,
+    "d_sae": 32768,
+    "activation_fn_str": "relu",
+    "apply_b_dec_to_input": true,
+    "finetuning_scaling_factor": false,
+    "context_size": 1024,
+    "model_name": "gemma-2-2b",
+    "hook_name": "blocks.8.hook_resid_post",
+    "hook_layer": 8,
+    "hook_head_index": null,
+    "prepend_bos": true,
+    "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B",
+    "dataset_trust_remote_code": true,
+    "normalize_activations": "none",
+    "dtype": "torch.bfloat16",
+    "device": "cuda",
+    "sae_lens_training_version": "5.5.1",
+    "activation_fn_kwargs": {
+      "k": 40
+    },
+    "neuronpedia_id": null,
+    "model_from_pretrained_kwargs": {
+      "center_writing_weights": false
+    },
+    "seqpos_slice": [
+      null
+    ]
+  },
+  "eval_result_unstructured": null
+}