Upload folder using huggingface_hub

L0/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 0,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 23.102449417114258,
+        "out": 3.8546135425567627
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L0/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2851325a82fcf4797fb18ac0d279d05ca10b619d5920a9edb2f4dc98a3355db
+size 57006482

L0/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00106,
+    "explained_variance": 0.93259,
+    "l1": 15.9,
+    "ground_truth_norm": 3.862,
+    "reconstructed_norm": 3.764,
+    "error_norm": 0.801,
+    "sparsity/below 1e-5": 3331,
+    "sparsity/below 1e-6": 3316,
+    "positivity": 1.0
+}

L1/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 1,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 27.244165420532227,
+        "out": 3.7663726806640625
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L1/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcd30b078958cf045da8d155e337a953dd97ed6ee760b86928dec33d623bd1b6
+size 57006482

L1/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.0011,
+    "explained_variance": 0.92292,
+    "l1": 16.14,
+    "ground_truth_norm": 3.762,
+    "reconstructed_norm": 3.65,
+    "error_norm": 0.8645,
+    "sparsity/below 1e-5": 2975,
+    "sparsity/below 1e-6": 2961,
+    "positivity": 1.0
+}

L10/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 10,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 144.09291076660156,
+        "out": 3.9614734649658203
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L10/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:54b1c9c7007f81d230ba61c52810ec8b3232dc5fe33a7e2171d1297edf82342a
+size 57006482

L10/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00277,
+    "explained_variance": 0.8377,
+    "l1": 16.73,
+    "ground_truth_norm": 3.964,
+    "reconstructed_norm": 3.734,
+    "error_norm": 1.2477,
+    "sparsity/below 1e-5": 3543,
+    "sparsity/below 1e-6": 3510,
+    "positivity": 1.0
+}

L11/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 11,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 150.25917053222656,
+        "out": 20.201923370361328
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L11/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:086852435fe7be016564297809b2083e7e2e036641fcf2dc4e7926e6aaa7b19c
+size 57006482

L11/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00404,
+    "explained_variance": 0.77293,
+    "l1": 18.28,
+    "ground_truth_norm": 20.238,
+    "reconstructed_norm": 20.157,
+    "error_norm": 1.5937,
+    "sparsity/below 1e-5": 4465,
+    "sparsity/below 1e-6": 4406,
+    "positivity": 1.0
+}

L2/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 2,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 27.967687606811523,
+        "out": 4.815014839172363
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L2/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c93793b9010c647284edb251923afc56ffae582effc72edf1ea0e1c2334d985
+size 57006482

L2/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00293,
+    "explained_variance": 0.88528,
+    "l1": 21.08,
+    "ground_truth_norm": 4.764,
+    "reconstructed_norm": 4.533,
+    "error_norm": 1.4217,
+    "sparsity/below 1e-5": 2958,
+    "sparsity/below 1e-6": 2950,
+    "positivity": 1.0
+}

L3/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 3,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 29.04673957824707,
+        "out": 7.051708698272705
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L3/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69691f0beacbf8f0f7b58e245d20f60f764cc445fc418510ae3d1c176a5667e9
+size 57006482

L3/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00703,
+    "explained_variance": 0.883,
+    "l1": 39.99,
+    "ground_truth_norm": 7.057,
+    "reconstructed_norm": 6.667,
+    "error_norm": 2.2736,
+    "sparsity/below 1e-5": 4204,
+    "sparsity/below 1e-6": 4204,
+    "positivity": 1.0
+}

L4/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 4,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 39.33174514770508,
+        "out": 5.237796306610107
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L4/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67a2bb6b113eb111dfb759d38f61fdc4bbf6e9e6e99c661608dc6d3fa46aa1b2
+size 57006482

L4/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00345,
+    "explained_variance": 0.88722,
+    "l1": 23.17,
+    "ground_truth_norm": 5.202,
+    "reconstructed_norm": 4.961,
+    "error_norm": 1.5414,
+    "sparsity/below 1e-5": 3891,
+    "sparsity/below 1e-6": 3887,
+    "positivity": 1.0
+}

L5/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 5,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 40.27893829345703,
+        "out": 5.156437873840332
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L5/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d430374109057795a15794d3e5d72fe6eb41cb79687e3d70e7703b4e000120de
+size 57006482

L5/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.0039,
+    "explained_variance": 0.88818,
+    "l1": 24.1,
+    "ground_truth_norm": 5.135,
+    "reconstructed_norm": 4.869,
+    "error_norm": 1.5905,
+    "sparsity/below 1e-5": 3800,
+    "sparsity/below 1e-6": 3796,
+    "positivity": 1.0
+}

L6/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 6,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 41.99013900756836,
+        "out": 4.518642902374268
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L6/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:608cd82d0fc3177f79ff8d2afcd49ccdd5eb14bb1c3cc733013a4f3f37deec5a
+size 57006482

L6/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00344,
+    "explained_variance": 0.85124,
+    "l1": 19.99,
+    "ground_truth_norm": 4.526,
+    "reconstructed_norm": 4.249,
+    "error_norm": 1.5198,
+    "sparsity/below 1e-5": 3257,
+    "sparsity/below 1e-6": 3251,
+    "positivity": 1.0
+}

L7/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 7,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 57.60716247558594,
+        "out": 5.574440956115723
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L7/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:061d14ad379b786671e815ebe16c3964cfcac5c0b956bcbb0e05c11f0145a443
+size 57006482

L7/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00393,
+    "explained_variance": 0.87354,
+    "l1": 26.84,
+    "ground_truth_norm": 5.58,
+    "reconstructed_norm": 5.312,
+    "error_norm": 1.6651,
+    "sparsity/below 1e-5": 4378,
+    "sparsity/below 1e-6": 4367,
+    "positivity": 1.0
+}

L8/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 8,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 86.12666320800781,
+        "out": 5.070123672485352
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L8/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5db36f3ba8ff080181ec1f368351c9e13021056ecce6b602347fa2c8088a2487
+size 57006482

L8/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.0032,
+    "explained_variance": 0.88431,
+    "l1": 22.02,
+    "ground_truth_norm": 5.037,
+    "reconstructed_norm": 4.824,
+    "error_norm": 1.3827,
+    "sparsity/below 1e-5": 3809,
+    "sparsity/below 1e-6": 3727,
+    "positivity": 1.0
+}

L9/config.json

@@ -0,0 +1,31 @@
+{
+    "d_qk_head": 128,
+    "d_ov_head": 1,
+    "n_qk_heads": 24,
+    "n_ov_heads": 6144,
+    "device": "cuda",
+    "dtype": "torch.float",
+    "virtual_kv_num": 0,
+    "use_z_relu": true,
+    "n_ctx": 256,
+    "layer": 9,
+    "model_name": "EleutherAI/pythia-160m",
+    "mode": "top_k",
+    "top_k": 64,
+    "avg_norm": {
+        "in": 117.01543426513672,
+        "out": 4.0516133308410645
+    },
+    "d_model": 768,
+    "attn_scale": 8.0,
+    "positional_embedding_type": "rotary",
+    "rotary_scale": 1,
+    "rotary_dim": 128,
+    "rotary_base": 10000,
+    "rotary_adjacent_pairs": false,
+    "use_NTK_by_parts_rope": false,
+    "NTK_by_parts_low_freq_factor": null,
+    "NTK_by_parts_high_freq_factor": null,
+    "NTK_by_parts_factor": null,
+    "old_context_len": null
+}

L9/final.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff7c8c57c2a245adec188f711e33c393da6f786e92908155b91ec059d65dec62
+size 57006482

L9/metrics.json

@@ -0,0 +1,11 @@
+{
+    "mse_loss": 0.00258,
+    "explained_variance": 0.87377,
+    "l1": 18.61,
+    "ground_truth_norm": 4.069,
+    "reconstructed_norm": 3.853,
+    "error_norm": 1.223,
+    "sparsity/below 1e-5": 3197,
+    "sparsity/below 1e-6": 3019,
+    "positivity": 1.0
+}