Upload folder using huggingface_hub

L2A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 2,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 25.82196044921875,
+        "out": 0.8914777040481567
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L2A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:292697deb870ed9e8ea3ff5db00f022f1181965d97a0db9a6b355465d5505413
+size 1614040466

L2A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 1.25998e-05,
+    "explained_variance": 0.88186,
+    "l1": 4.18,
+    "ground_truth_norm": 0.889,
+    "reconstructed_norm": 0.862,
+    "error_norm": 0.2141,
+    "sparsity/below 1e-5": 16279,
+    "sparsity/below 1e-6": 15533,
+    "positivity": 0.54269,
+    "ov_head_live_count": {
+        "0": 312,
+        "1": 338,
+        "2": 186,
+        "3": 141,
+        "4": 450,
+        "5": 439,
+        "6": 173,
+        "7": 377,
+        "8": 167,
+        "9": 128,
+        "10": 332,
+        "11": 349,
+        "12": 250,
+        "13": 171,
+        "14": 422,
+        "15": 303,
+        "16": 429,
+        "17": 188,
+        "18": 65,
+        "19": 360,
+        "20": 321,
+        "21": 215,
+        "22": 432,
+        "23": 426,
+        "24": 285,
+        "25": 167,
+        "26": 345,
+        "27": 167,
+        "28": 350,
+        "29": 461,
+        "30": 262,
+        "31": 271,
+        "32": 79,
+        "33": 134,
+        "34": 179,
+        "35": 356,
+        "36": 329,
+        "37": 433,
+        "38": 269,
+        "39": 84,
+        "40": 243,
+        "41": 443,
+        "42": 348,
+        "43": 196,
+        "44": 169,
+        "45": 333,
+        "46": 291,
+        "47": 171,
+        "48": 310,
+        "49": 291,
+        "50": 375,
+        "51": 327,
+        "52": 305,
+        "53": 92,
+        "54": 262,
+        "55": 207,
+        "56": 236,
+        "57": 328,
+        "58": 165,
+        "59": 395,
+        "60": 211,
+        "61": 451,
+        "62": 68,
+        "63": 421
+    }
+}

L3A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 3,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 22.983654022216797,
+        "out": 1.1687864065170288
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L3A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96f03145dca4a61a1f544ef7ce14602c6492bd6c6c2350c784b4b0e70451446e
+size 1614040466

L3A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 4.04951e-05,
+    "explained_variance": 0.82606,
+    "l1": 5.99,
+    "ground_truth_norm": 1.16,
+    "reconstructed_norm": 1.089,
+    "error_norm": 0.3918,
+    "sparsity/below 1e-5": 15990,
+    "sparsity/below 1e-6": 15516,
+    "positivity": 0.54037,
+    "ov_head_live_count": {
+        "0": 290,
+        "1": 176,
+        "2": 267,
+        "3": 238,
+        "4": 334,
+        "5": 57,
+        "6": 472,
+        "7": 370,
+        "8": 150,
+        "9": 158,
+        "10": 340,
+        "11": 212,
+        "12": 254,
+        "13": 425,
+        "14": 233,
+        "15": 418,
+        "16": 465,
+        "17": 396,
+        "18": 458,
+        "19": 64,
+        "20": 346,
+        "21": 162,
+        "22": 166,
+        "23": 367,
+        "24": 270,
+        "25": 273,
+        "26": 185,
+        "27": 236,
+        "28": 230,
+        "29": 343,
+        "30": 410,
+        "31": 238,
+        "32": 207,
+        "33": 26,
+        "34": 129,
+        "35": 148,
+        "36": 104,
+        "37": 305,
+        "38": 478,
+        "39": 364,
+        "40": 177,
+        "41": 434,
+        "42": 360,
+        "43": 491,
+        "44": 162,
+        "45": 342,
+        "46": 299,
+        "47": 363,
+        "48": 60,
+        "49": 254,
+        "50": 299,
+        "51": 243,
+        "52": 379,
+        "53": 91,
+        "54": 364,
+        "55": 449,
+        "56": 412,
+        "57": 112,
+        "58": 265,
+        "59": 188,
+        "60": 55,
+        "61": 342,
+        "62": 311,
+        "63": 491
+    }
+}

L4A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 4,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 22.783418655395508,
+        "out": 1.4010306596755981
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L4A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f02d1943cbe67836523652be7175806ef2af2a3765bb3940864e6c58162c778
+size 1614040466

L4A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 7.74775e-05,
+    "explained_variance": 0.80203,
+    "l1": 7.87,
+    "ground_truth_norm": 1.413,
+    "reconstructed_norm": 1.301,
+    "error_norm": 0.5429,
+    "sparsity/below 1e-5": 18830,
+    "sparsity/below 1e-6": 18220,
+    "positivity": 0.45062,
+    "ov_head_live_count": {
+        "0": 327,
+        "1": 150,
+        "2": 366,
+        "3": 310,
+        "4": 107,
+        "5": 407,
+        "6": 80,
+        "7": 52,
+        "8": 210,
+        "9": 213,
+        "10": 228,
+        "11": 212,
+        "12": 214,
+        "13": 360,
+        "14": 273,
+        "15": 202,
+        "16": 174,
+        "17": 273,
+        "18": 45,
+        "19": 165,
+        "20": 194,
+        "21": 173,
+        "22": 177,
+        "23": 238,
+        "24": 383,
+        "25": 198,
+        "26": 193,
+        "27": 415,
+        "28": 297,
+        "29": 312,
+        "30": 251,
+        "31": 430,
+        "32": 184,
+        "33": 38,
+        "34": 152,
+        "35": 150,
+        "36": 224,
+        "37": 303,
+        "38": 221,
+        "39": 195,
+        "40": 148,
+        "41": 128,
+        "42": 71,
+        "43": 433,
+        "44": 277,
+        "45": 204,
+        "46": 249,
+        "47": 100,
+        "48": 126,
+        "49": 305,
+        "50": 243,
+        "51": 270,
+        "52": 236,
+        "53": 387,
+        "54": 272,
+        "55": 180,
+        "56": 161,
+        "57": 233,
+        "58": 341,
+        "59": 331,
+        "60": 264,
+        "61": 117,
+        "62": 292,
+        "63": 302
+    }
+}