Upload folder using huggingface_hub

L12A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 12,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 27.36377716064453,
+        "out": 2.912698745727539
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L12A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e94c9348a6c6e4382ce1f141a3be0fead61c0c81f073144900e0bac8692570be
+size 1614040466

L12A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0006444874,
+    "explained_variance": 0.65362,
+    "l1": 16.72,
+    "ground_truth_norm": 2.933,
+    "reconstructed_norm": 2.46,
+    "error_norm": 1.5802,
+    "sparsity/below 1e-5": 24648,
+    "sparsity/below 1e-6": 24548,
+    "positivity": 0.2558,
+    "ov_head_live_count": {
+        "0": 113,
+        "1": 3,
+        "2": 28,
+        "3": 61,
+        "4": 361,
+        "5": 79,
+        "6": 180,
+        "7": 20,
+        "8": 215,
+        "9": 12,
+        "10": 97,
+        "11": 136,
+        "12": 146,
+        "13": 89,
+        "14": 13,
+        "15": 30,
+        "16": 5,
+        "17": 20,
+        "18": 183,
+        "19": 7,
+        "20": 191,
+        "21": 216,
+        "22": 2,
+        "23": 21,
+        "24": 166,
+        "25": 104,
+        "26": 334,
+        "27": 418,
+        "28": 368,
+        "29": 35,
+        "30": 306,
+        "31": 383,
+        "32": 166,
+        "33": 394,
+        "34": 129,
+        "35": 64,
+        "36": 418,
+        "37": 29,
+        "38": 10,
+        "39": 130,
+        "40": 122,
+        "41": 175,
+        "42": 8,
+        "43": 124,
+        "44": 15,
+        "45": 20,
+        "46": 13,
+        "47": 198,
+        "48": 308,
+        "49": 141,
+        "50": 27,
+        "51": 166,
+        "52": 85,
+        "53": 8,
+        "54": 247,
+        "55": 142,
+        "56": 31,
+        "57": 18,
+        "58": 47,
+        "59": 80,
+        "60": 277,
+        "61": 73,
+        "62": 157,
+        "63": 218
+    }
+}

L13A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 13,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 31.526508331298828,
+        "out": 3.1382012367248535
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L13A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8bf691e58b635c9c5a20488926004b6c872f6a4a52693fb278b2f00f5f96a80
+size 1614040466

L13A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0007835636,
+    "explained_variance": 0.68137,
+    "l1": 18.08,
+    "ground_truth_norm": 3.16,
+    "reconstructed_norm": 2.63,
+    "error_norm": 1.7362,
+    "sparsity/below 1e-5": 25078,
+    "sparsity/below 1e-6": 25001,
+    "positivity": 0.24667,
+    "ov_head_live_count": {
+        "0": 180,
+        "1": 68,
+        "2": 279,
+        "3": 133,
+        "4": 12,
+        "5": 123,
+        "6": 120,
+        "7": 55,
+        "8": 189,
+        "9": 117,
+        "10": 410,
+        "11": 41,
+        "12": 171,
+        "13": 12,
+        "14": 35,
+        "15": 255,
+        "16": 40,
+        "17": 3,
+        "18": 9,
+        "19": 38,
+        "20": 207,
+        "21": 354,
+        "22": 223,
+        "23": 450,
+        "24": 27,
+        "25": 31,
+        "26": 65,
+        "27": 21,
+        "28": 217,
+        "29": 63,
+        "30": 278,
+        "31": 25,
+        "32": 197,
+        "33": 368,
+        "34": 101,
+        "35": 64,
+        "36": 310,
+        "37": 32,
+        "38": 174,
+        "39": 286,
+        "40": 109,
+        "41": 101,
+        "42": 7,
+        "43": 147,
+        "44": 26,
+        "45": 38,
+        "46": 55,
+        "47": 7,
+        "48": 351,
+        "49": 93,
+        "50": 10,
+        "51": 14,
+        "52": 3,
+        "53": 179,
+        "54": 129,
+        "55": 182,
+        "56": 23,
+        "57": 65,
+        "58": 37,
+        "59": 180,
+        "60": 82,
+        "61": 117,
+        "62": 37,
+        "63": 308
+    }
+}

L14A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 14,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 32.4897575378418,
+        "out": 3.191192865371704
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L14A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57feb9549ec962084f9978ff99b2c3ec6ec4c5271e6bd270a00280a5442ff483
+size 1614040466

L14A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0007458544,
+    "explained_variance": 0.71227,
+    "l1": 20.76,
+    "ground_truth_norm": 3.223,
+    "reconstructed_norm": 2.733,
+    "error_norm": 1.6916,
+    "sparsity/below 1e-5": 26050,
+    "sparsity/below 1e-6": 26003,
+    "positivity": 0.20981,
+    "ov_head_live_count": {
+        "0": 17,
+        "1": 107,
+        "2": 53,
+        "3": 146,
+        "4": 3,
+        "5": 114,
+        "6": 19,
+        "7": 20,
+        "8": 92,
+        "9": 123,
+        "10": 178,
+        "11": 182,
+        "12": 230,
+        "13": 181,
+        "14": 45,
+        "15": 64,
+        "16": 310,
+        "17": 9,
+        "18": 15,
+        "19": 27,
+        "20": 154,
+        "21": 94,
+        "22": 117,
+        "23": 164,
+        "24": 189,
+        "25": 64,
+        "26": 190,
+        "27": 90,
+        "28": 191,
+        "29": 178,
+        "30": 47,
+        "31": 151,
+        "32": 5,
+        "33": 264,
+        "34": 82,
+        "35": 121,
+        "36": 86,
+        "37": 17,
+        "38": 55,
+        "39": 8,
+        "40": 410,
+        "41": 107,
+        "42": 51,
+        "43": 44,
+        "44": 233,
+        "45": 26,
+        "46": 98,
+        "47": 34,
+        "48": 95,
+        "49": 99,
+        "50": 162,
+        "51": 150,
+        "52": 298,
+        "53": 10,
+        "54": 85,
+        "55": 20,
+        "56": 69,
+        "57": 235,
+        "58": 38,
+        "59": 187,
+        "60": 18,
+        "61": 107,
+        "62": 66,
+        "63": 31
+    }
+}

L15A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 15,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 30.77876853942871,
+        "out": 3.105498790740967
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L15A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82ba1571ed03bc6e7ae938b992bda3eba7a6bf0f93bf2502ff77c7d9ad326e11
+size 1614040466

L15A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0007111974,
+    "explained_variance": 0.70865,
+    "l1": 18.94,
+    "ground_truth_norm": 3.159,
+    "reconstructed_norm": 2.682,
+    "error_norm": 1.6552,
+    "sparsity/below 1e-5": 22472,
+    "sparsity/below 1e-6": 22379,
+    "positivity": 0.32211,
+    "ov_head_live_count": {
+        "0": 302,
+        "1": 81,
+        "2": 361,
+        "3": 286,
+        "4": 184,
+        "5": 178,
+        "6": 258,
+        "7": 84,
+        "8": 88,
+        "9": 419,
+        "10": 113,
+        "11": 161,
+        "12": 365,
+        "13": 175,
+        "14": 327,
+        "15": 154,
+        "16": 355,
+        "17": 289,
+        "18": 244,
+        "19": 50,
+        "20": 93,
+        "21": 39,
+        "22": 144,
+        "23": 16,
+        "24": 194,
+        "25": 101,
+        "26": 122,
+        "27": 43,
+        "28": 35,
+        "29": 59,
+        "30": 123,
+        "31": 292,
+        "32": 281,
+        "33": 291,
+        "34": 107,
+        "35": 380,
+        "36": 284,
+        "37": 74,
+        "38": 108,
+        "39": 12,
+        "40": 212,
+        "41": 272,
+        "42": 52,
+        "43": 200,
+        "44": 71,
+        "45": 98,
+        "46": 72,
+        "47": 2,
+        "48": 198,
+        "49": 86,
+        "50": 208,
+        "51": 69,
+        "52": 145,
+        "53": 141,
+        "54": 10,
+        "55": 149,
+        "56": 82,
+        "57": 269,
+        "58": 126,
+        "59": 26,
+        "60": 339,
+        "61": 151,
+        "62": 121,
+        "63": 184
+    }
+}

L16A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 16,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 31.657766342163086,
+        "out": 3.292898416519165
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L16A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ca2047752006f4fed44dc1272e1f8b9a4e12e6750cc1b2efae923ab53ad897e
+size 1614040466

L16A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0006912273,
+    "explained_variance": 0.73999,
+    "l1": 20.49,
+    "ground_truth_norm": 3.339,
+    "reconstructed_norm": 2.904,
+    "error_norm": 1.6263,
+    "sparsity/below 1e-5": 22885,
+    "sparsity/below 1e-6": 22814,
+    "positivity": 0.3063,
+    "ov_head_live_count": {
+        "0": 216,
+        "1": 304,
+        "2": 7,
+        "3": 287,
+        "4": 379,
+        "5": 244,
+        "6": 95,
+        "7": 110,
+        "8": 121,
+        "9": 75,
+        "10": 3,
+        "11": 290,
+        "12": 72,
+        "13": 57,
+        "14": 104,
+        "15": 140,
+        "16": 29,
+        "17": 125,
+        "18": 257,
+        "19": 305,
+        "20": 94,
+        "21": 296,
+        "22": 154,
+        "23": 92,
+        "24": 179,
+        "25": 278,
+        "26": 70,
+        "27": 99,
+        "28": 176,
+        "29": 146,
+        "30": 56,
+        "31": 57,
+        "32": 6,
+        "33": 269,
+        "34": 194,
+        "35": 172,
+        "36": 170,
+        "37": 150,
+        "38": 3,
+        "39": 46,
+        "40": 83,
+        "41": 77,
+        "42": 107,
+        "43": 11,
+        "44": 2,
+        "45": 208,
+        "46": 352,
+        "47": 16,
+        "48": 364,
+        "49": 88,
+        "50": 247,
+        "51": 54,
+        "52": 141,
+        "53": 182,
+        "54": 358,
+        "55": 305,
+        "56": 38,
+        "57": 140,
+        "58": 186,
+        "59": 71,
+        "60": 319,
+        "61": 375,
+        "62": 90,
+        "63": 296
+    }
+}

L17A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 17,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 31.074840545654297,
+        "out": 3.057485818862915
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L17A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60c31a6ebb759b91b11aa70f4b80cb7b3135f745783599134d3df77b72b64017
+size 1614040466

L17A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0005549974,
+    "explained_variance": 0.75869,
+    "l1": 19.07,
+    "ground_truth_norm": 3.08,
+    "reconstructed_norm": 2.707,
+    "error_norm": 1.4556,
+    "sparsity/below 1e-5": 20583,
+    "sparsity/below 1e-6": 20264,
+    "positivity": 0.39352,
+    "ov_head_live_count": {
+        "0": 232,
+        "1": 170,
+        "2": 219,
+        "3": 309,
+        "4": 196,
+        "5": 345,
+        "6": 351,
+        "7": 127,
+        "8": 174,
+        "9": 63,
+        "10": 150,
+        "11": 287,
+        "12": 383,
+        "13": 268,
+        "14": 79,
+        "15": 64,
+        "16": 21,
+        "17": 94,
+        "18": 156,
+        "19": 148,
+        "20": 256,
+        "21": 136,
+        "22": 282,
+        "23": 73,
+        "24": 373,
+        "25": 354,
+        "26": 265,
+        "27": 85,
+        "28": 44,
+        "29": 237,
+        "30": 49,
+        "31": 71,
+        "32": 374,
+        "33": 249,
+        "34": 215,
+        "35": 44,
+        "36": 337,
+        "37": 118,
+        "38": 387,
+        "39": 306,
+        "40": 119,
+        "41": 361,
+        "42": 361,
+        "43": 285,
+        "44": 278,
+        "45": 130,
+        "46": 318,
+        "47": 135,
+        "48": 391,
+        "49": 172,
+        "50": 268,
+        "51": 61,
+        "52": 232,
+        "53": 88,
+        "54": 363,
+        "55": 79,
+        "56": 352,
+        "57": 57,
+        "58": 205,
+        "59": 115,
+        "60": 51,
+        "61": 70,
+        "62": 197,
+        "63": 116
+    }
+}

L18A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 18,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 31.758373260498047,
+        "out": 2.5856893062591553
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L18A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5dc42a30ce6e391e61b142aafba75c0aa18895756149da495f7243341430f04
+size 1614040466

L18A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0003417022,
+    "explained_variance": 0.78765,
+    "l1": 16.81,
+    "ground_truth_norm": 2.625,
+    "reconstructed_norm": 2.361,
+    "error_norm": 1.1308,
+    "sparsity/below 1e-5": 18686,
+    "sparsity/below 1e-6": 17927,
+    "positivity": 0.48187,
+    "ov_head_live_count": {
+        "0": 460,
+        "1": 349,
+        "2": 470,
+        "3": 253,
+        "4": 337,
+        "5": 343,
+        "6": 99,
+        "7": 414,
+        "8": 139,
+        "9": 172,
+        "10": 185,
+        "11": 43,
+        "12": 418,
+        "13": 387,
+        "14": 265,
+        "15": 419,
+        "16": 170,
+        "17": 48,
+        "18": 173,
+        "19": 255,
+        "20": 297,
+        "21": 196,
+        "22": 118,
+        "23": 145,
+        "24": 143,
+        "25": 402,
+        "26": 151,
+        "27": 451,
+        "28": 76,
+        "29": 345,
+        "30": 254,
+        "31": 208,
+        "32": 213,
+        "33": 164,
+        "34": 299,
+        "35": 163,
+        "36": 309,
+        "37": 311,
+        "38": 197,
+        "39": 227,
+        "40": 221,
+        "41": 315,
+        "42": 127,
+        "43": 389,
+        "44": 272,
+        "45": 160,
+        "46": 74,
+        "47": 54,
+        "48": 423,
+        "49": 127,
+        "50": 317,
+        "51": 59,
+        "52": 395,
+        "53": 261,
+        "54": 406,
+        "55": 306,
+        "56": 165,
+        "57": 276,
+        "58": 96,
+        "59": 347,
+        "60": 170,
+        "61": 339,
+        "62": 394,
+        "63": 29
+    }
+}

L19A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 19,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 31.919294357299805,
+        "out": 2.284219980239868
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L19A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1c138b05a313c4085b599ba89e1fd445b66f23ff0ec12023139180cecb7a3ce
+size 1614040466

L19A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0003078198,
+    "explained_variance": 0.76338,
+    "l1": 15.76,
+    "ground_truth_norm": 2.332,
+    "reconstructed_norm": 2.067,
+    "error_norm": 1.0594,
+    "sparsity/below 1e-5": 16110,
+    "sparsity/below 1e-6": 14863,
+    "positivity": 0.56973,
+    "ov_head_live_count": {
+        "0": 409,
+        "1": 263,
+        "2": 416,
+        "3": 195,
+        "4": 357,
+        "5": 319,
+        "6": 162,
+        "7": 444,
+        "8": 244,
+        "9": 109,
+        "10": 219,
+        "11": 417,
+        "12": 278,
+        "13": 299,
+        "14": 350,
+        "15": 420,
+        "16": 209,
+        "17": 39,
+        "18": 362,
+        "19": 331,
+        "20": 204,
+        "21": 195,
+        "22": 415,
+        "23": 382,
+        "24": 350,
+        "25": 106,
+        "26": 342,
+        "27": 201,
+        "28": 334,
+        "29": 324,
+        "30": 390,
+        "31": 385,
+        "32": 412,
+        "33": 373,
+        "34": 449,
+        "35": 224,
+        "36": 296,
+        "37": 345,
+        "38": 86,
+        "39": 422,
+        "40": 327,
+        "41": 67,
+        "42": 447,
+        "43": 159,
+        "44": 278,
+        "45": 251,
+        "46": 303,
+        "47": 357,
+        "48": 147,
+        "49": 48,
+        "50": 301,
+        "51": 154,
+        "52": 177,
+        "53": 252,
+        "54": 435,
+        "55": 173,
+        "56": 445,
+        "57": 379,
+        "58": 175,
+        "59": 359,
+        "60": 399,
+        "61": 364,
+        "62": 251,
+        "63": 344
+    }
+}

L20A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 20,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 32.97787094116211,
+        "out": 2.3633012771606445
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L20A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63b1bc4cde7614df26b9cadb4d6cb29796862ee6167c444359d1f6cc4f6f98b2
+size 1614040466

L20A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0003548504,
+    "explained_variance": 0.75766,
+    "l1": 16.56,
+    "ground_truth_norm": 2.379,
+    "reconstructed_norm": 2.083,
+    "error_norm": 1.1285,
+    "sparsity/below 1e-5": 17100,
+    "sparsity/below 1e-6": 16072,
+    "positivity": 0.5462,
+    "ov_head_live_count": {
+        "0": 419,
+        "1": 199,
+        "2": 234,
+        "3": 327,
+        "4": 80,
+        "5": 479,
+        "6": 245,
+        "7": 362,
+        "8": 93,
+        "9": 449,
+        "10": 274,
+        "11": 78,
+        "12": 298,
+        "13": 315,
+        "14": 406,
+        "15": 257,
+        "16": 353,
+        "17": 162,
+        "18": 330,
+        "19": 359,
+        "20": 139,
+        "21": 102,
+        "22": 371,
+        "23": 330,
+        "24": 444,
+        "25": 309,
+        "26": 276,
+        "27": 160,
+        "28": 208,
+        "29": 345,
+        "30": 39,
+        "31": 93,
+        "32": 264,
+        "33": 58,
+        "34": 221,
+        "35": 229,
+        "36": 411,
+        "37": 212,
+        "38": 323,
+        "39": 335,
+        "40": 282,
+        "41": 327,
+        "42": 430,
+        "43": 308,
+        "44": 298,
+        "45": 420,
+        "46": 420,
+        "47": 279,
+        "48": 179,
+        "49": 193,
+        "50": 146,
+        "51": 374,
+        "52": 449,
+        "53": 423,
+        "54": 261,
+        "55": 311,
+        "56": 462,
+        "57": 118,
+        "58": 398,
+        "59": 107,
+        "60": 147,
+        "61": 341,
+        "62": 273,
+        "63": 364
+    }
+}

L21A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 21,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 32.402626037597656,
+        "out": 3.294581651687622
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L21A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:551b4bd4081bce1b00c2cf43eeaefb76498251abd586663728edcbb31e78c91a
+size 1614040466

L21A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0005338052,
+    "explained_variance": 0.80594,
+    "l1": 21.32,
+    "ground_truth_norm": 3.356,
+    "reconstructed_norm": 3.039,
+    "error_norm": 1.4049,
+    "sparsity/below 1e-5": 19748,
+    "sparsity/below 1e-6": 19430,
+    "positivity": 0.44174,
+    "ov_head_live_count": {
+        "0": 85,
+        "1": 155,
+        "2": 197,
+        "3": 284,
+        "4": 319,
+        "5": 171,
+        "6": 218,
+        "7": 375,
+        "8": 86,
+        "9": 91,
+        "10": 352,
+        "11": 186,
+        "12": 356,
+        "13": 116,
+        "14": 289,
+        "15": 77,
+        "16": 220,
+        "17": 409,
+        "18": 84,
+        "19": 106,
+        "20": 89,
+        "21": 345,
+        "22": 311,
+        "23": 271,
+        "24": 181,
+        "25": 138,
+        "26": 282,
+        "27": 72,
+        "28": 124,
+        "29": 292,
+        "30": 199,
+        "31": 219,
+        "32": 230,
+        "33": 137,
+        "34": 362,
+        "35": 258,
+        "36": 226,
+        "37": 260,
+        "38": 287,
+        "39": 199,
+        "40": 315,
+        "41": 109,
+        "42": 311,
+        "43": 303,
+        "44": 281,
+        "45": 330,
+        "46": 390,
+        "47": 175,
+        "48": 307,
+        "49": 224,
+        "50": 292,
+        "51": 222,
+        "52": 446,
+        "53": 276,
+        "54": 262,
+        "55": 57,
+        "56": 209,
+        "57": 90,
+        "58": 151,
+        "59": 216,
+        "60": 184,
+        "61": 87,
+        "62": 362,
+        "63": 218
+    }
+}

L22A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 22,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 34.090354919433594,
+        "out": 2.561861753463745
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L22A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:632b82ac74d6c1a42d3da0b866e87d4ce2efa80bfde820f5144ff7d86084a830
+size 1614040466

L22A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0004232735,
+    "explained_variance": 0.76295,
+    "l1": 17.34,
+    "ground_truth_norm": 2.654,
+    "reconstructed_norm": 2.344,
+    "error_norm": 1.2268,
+    "sparsity/below 1e-5": 16999,
+    "sparsity/below 1e-6": 16475,
+    "positivity": 0.52084,
+    "ov_head_live_count": {
+        "0": 184,
+        "1": 358,
+        "2": 169,
+        "3": 436,
+        "4": 375,
+        "5": 156,
+        "6": 52,
+        "7": 228,
+        "8": 286,
+        "9": 27,
+        "10": 274,
+        "11": 254,
+        "12": 38,
+        "13": 395,
+        "14": 154,
+        "15": 453,
+        "16": 430,
+        "17": 426,
+        "18": 321,
+        "19": 396,
+        "20": 119,
+        "21": 307,
+        "22": 284,
+        "23": 277,
+        "24": 66,
+        "25": 261,
+        "26": 156,
+        "27": 447,
+        "28": 368,
+        "29": 329,
+        "30": 400,
+        "31": 360,
+        "32": 370,
+        "33": 363,
+        "34": 91,
+        "35": 264,
+        "36": 145,
+        "37": 247,
+        "38": 38,
+        "39": 124,
+        "40": 416,
+        "41": 252,
+        "42": 422,
+        "43": 274,
+        "44": 261,
+        "45": 84,
+        "46": 260,
+        "47": 295,
+        "48": 268,
+        "49": 324,
+        "50": 161,
+        "51": 176,
+        "52": 86,
+        "53": 219,
+        "54": 369,
+        "55": 285,
+        "56": 250,
+        "57": 395,
+        "58": 382,
+        "59": 273,
+        "60": 206,
+        "61": 388,
+        "62": 418,
+        "63": 175
+    }
+}

L23A/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 256,
+    "d_ov_head": 1,
+    "n_qk_heads": 64,
+    "n_ov_heads": 32768,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 1024,
+    "layer": 23,
+    "model_name": "meta-llama/Llama-3.1-8B",
+    "mode": "top_k",
+    "top_k": 128,
+    "avg_norm": {
+        "in": 33.478187561035156,
+        "out": 2.9890081882476807
+    },
+    "d_model": 4096,
+    "attn_scale": 11.313708498984761,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 256,
+    "rotary_base": 500000.0,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": true,
+    "NTK_by_parts_low_freq_factor": 1.0,
+    "NTK_by_parts_high_freq_factor": 4.0,
+    "NTK_by_parts_factor": 8.0,
+    "old_context_len": 8192
+}

L23A/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6955da6d7a1e9375d0b67241d9f99f1143f752d1aa25ed4de710927f0746dacc
+size 1614040466

L23A/metrics.json

@@ -0,0 +1,77 @@
+{
+    "mse_loss": 0.0004005697,
+    "explained_variance": 0.80796,
+    "l1": 18.76,
+    "ground_truth_norm": 2.916,
+    "reconstructed_norm": 2.651,
+    "error_norm": 1.1865,
+    "sparsity/below 1e-5": 15034,
+    "sparsity/below 1e-6": 14151,
+    "positivity": 0.64301,
+    "ov_head_live_count": {
+        "0": 292,
+        "1": 314,
+        "2": 155,
+        "3": 317,
+        "4": 338,
+        "5": 308,
+        "6": 276,
+        "7": 123,
+        "8": 436,
+        "9": 274,
+        "10": 448,
+        "11": 430,
+        "12": 355,
+        "13": 341,
+        "14": 112,
+        "15": 192,
+        "16": 180,
+        "17": 395,
+        "18": 169,
+        "19": 414,
+        "20": 395,
+        "21": 404,
+        "22": 309,
+        "23": 490,
+        "24": 437,
+        "25": 419,
+        "26": 411,
+        "27": 131,
+        "28": 267,
+        "29": 364,
+        "30": 119,
+        "31": 471,
+        "32": 351,
+        "33": 375,
+        "34": 169,
+        "35": 448,
+        "36": 493,
+        "37": 270,
+        "38": 176,
+        "39": 339,
+        "40": 182,
+        "41": 336,
+        "42": 357,
+        "43": 367,
+        "44": 478,
+        "45": 335,
+        "46": 407,
+        "47": 498,
+        "48": 97,
+        "49": 433,
+        "50": 414,
+        "51": 423,
+        "52": 342,
+        "53": 111,
+        "54": 389,
+        "55": 344,
+        "56": 269,
+        "57": 379,
+        "58": 273,
+        "59": 358,
+        "60": 411,
+        "61": 443,
+        "62": 229,
+        "63": 488
+    }
+}