Upload folder using huggingface_hub

L0Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 0,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 23.102449417114258,
-        "out": 3.8546135425567627
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L0Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a2851325a82fcf4797fb18ac0d279d05ca10b619d5920a9edb2f4dc98a3355db
-size 57006482

L0Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00106,
-    "explained_variance": 0.93259,
-    "l1": 15.9,
-    "ground_truth_norm": 3.862,
-    "reconstructed_norm": 3.764,
-    "error_norm": 0.801,
-    "sparsity/below 1e-5": 3331,
-    "sparsity/below 1e-6": 3316,
-    "positivity": 1.0
-}

L10Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 10,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 144.09291076660156,
-        "out": 3.9614734649658203
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L10Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:54b1c9c7007f81d230ba61c52810ec8b3232dc5fe33a7e2171d1297edf82342a
-size 57006482

L10Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00277,
-    "explained_variance": 0.8377,
-    "l1": 16.73,
-    "ground_truth_norm": 3.964,
-    "reconstructed_norm": 3.734,
-    "error_norm": 1.2477,
-    "sparsity/below 1e-5": 3543,
-    "sparsity/below 1e-6": 3510,
-    "positivity": 1.0
-}

L11Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 11,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 150.25917053222656,
-        "out": 20.201923370361328
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L11Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:086852435fe7be016564297809b2083e7e2e036641fcf2dc4e7926e6aaa7b19c
-size 57006482

L11Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00404,
-    "explained_variance": 0.77293,
-    "l1": 18.28,
-    "ground_truth_norm": 20.238,
-    "reconstructed_norm": 20.157,
-    "error_norm": 1.5937,
-    "sparsity/below 1e-5": 4465,
-    "sparsity/below 1e-6": 4406,
-    "positivity": 1.0
-}

L1Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 1,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 27.244165420532227,
-        "out": 3.7663726806640625
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L1Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:dcd30b078958cf045da8d155e337a953dd97ed6ee760b86928dec33d623bd1b6
-size 57006482

L1Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.0011,
-    "explained_variance": 0.92292,
-    "l1": 16.14,
-    "ground_truth_norm": 3.762,
-    "reconstructed_norm": 3.65,
-    "error_norm": 0.8645,
-    "sparsity/below 1e-5": 2975,
-    "sparsity/below 1e-6": 2961,
-    "positivity": 1.0
-}

L2Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 2,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 27.967687606811523,
-        "out": 4.815014839172363
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L2Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7c93793b9010c647284edb251923afc56ffae582effc72edf1ea0e1c2334d985
-size 57006482

L2Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00293,
-    "explained_variance": 0.88528,
-    "l1": 21.08,
-    "ground_truth_norm": 4.764,
-    "reconstructed_norm": 4.533,
-    "error_norm": 1.4217,
-    "sparsity/below 1e-5": 2958,
-    "sparsity/below 1e-6": 2950,
-    "positivity": 1.0
-}

L3Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 3,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 29.04673957824707,
-        "out": 7.051708698272705
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L3Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:69691f0beacbf8f0f7b58e245d20f60f764cc445fc418510ae3d1c176a5667e9
-size 57006482

L3Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00703,
-    "explained_variance": 0.883,
-    "l1": 39.99,
-    "ground_truth_norm": 7.057,
-    "reconstructed_norm": 6.667,
-    "error_norm": 2.2736,
-    "sparsity/below 1e-5": 4204,
-    "sparsity/below 1e-6": 4204,
-    "positivity": 1.0
-}

L4Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 4,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 39.33174514770508,
-        "out": 5.237796306610107
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L4Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:67a2bb6b113eb111dfb759d38f61fdc4bbf6e9e6e99c661608dc6d3fa46aa1b2
-size 57006482

L4Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00345,
-    "explained_variance": 0.88722,
-    "l1": 23.17,
-    "ground_truth_norm": 5.202,
-    "reconstructed_norm": 4.961,
-    "error_norm": 1.5414,
-    "sparsity/below 1e-5": 3891,
-    "sparsity/below 1e-6": 3887,
-    "positivity": 1.0
-}

L5Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 5,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 40.27893829345703,
-        "out": 5.156437873840332
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L5Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d430374109057795a15794d3e5d72fe6eb41cb79687e3d70e7703b4e000120de
-size 57006482

L5Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.0039,
-    "explained_variance": 0.88818,
-    "l1": 24.1,
-    "ground_truth_norm": 5.135,
-    "reconstructed_norm": 4.869,
-    "error_norm": 1.5905,
-    "sparsity/below 1e-5": 3800,
-    "sparsity/below 1e-6": 3796,
-    "positivity": 1.0
-}

L6Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 6,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 41.99013900756836,
-        "out": 4.518642902374268
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L6Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:608cd82d0fc3177f79ff8d2afcd49ccdd5eb14bb1c3cc733013a4f3f37deec5a
-size 57006482

L6Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00344,
-    "explained_variance": 0.85124,
-    "l1": 19.99,
-    "ground_truth_norm": 4.526,
-    "reconstructed_norm": 4.249,
-    "error_norm": 1.5198,
-    "sparsity/below 1e-5": 3257,
-    "sparsity/below 1e-6": 3251,
-    "positivity": 1.0
-}

L7Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 7,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 57.60716247558594,
-        "out": 5.574440956115723
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L7Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:061d14ad379b786671e815ebe16c3964cfcac5c0b956bcbb0e05c11f0145a443
-size 57006482

L7Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00393,
-    "explained_variance": 0.87354,
-    "l1": 26.84,
-    "ground_truth_norm": 5.58,
-    "reconstructed_norm": 5.312,
-    "error_norm": 1.6651,
-    "sparsity/below 1e-5": 4378,
-    "sparsity/below 1e-6": 4367,
-    "positivity": 1.0
-}

L8Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 8,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 86.12666320800781,
-        "out": 5.070123672485352
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L8Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:5db36f3ba8ff080181ec1f368351c9e13021056ecce6b602347fa2c8088a2487
-size 57006482

L8Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.0032,
-    "explained_variance": 0.88431,
-    "l1": 22.02,
-    "ground_truth_norm": 5.037,
-    "reconstructed_norm": 4.824,
-    "error_norm": 1.3827,
-    "sparsity/below 1e-5": 3809,
-    "sparsity/below 1e-6": 3727,
-    "positivity": 1.0
-}

L9Lorsa/config.json

@@ -1,31 +0,0 @@
-{
-    "d_qk_head": 128,
-    "d_ov_head": 1,
-    "n_qk_heads": 24,
-    "n_ov_heads": 6144,
-    "device": "cuda",
-    "dtype": "torch.float",
-    "virtual_kv_num": 0,
-    "use_z_relu": true,
-    "n_ctx": 256,
-    "layer": 9,
-    "model_name": "EleutherAI/pythia-160m",
-    "mode": "top_k",
-    "top_k": 64,
-    "avg_norm": {
-        "in": 117.01543426513672,
-        "out": 4.0516133308410645
-    },
-    "d_model": 768,
-    "attn_scale": 8.0,
-    "positional_embedding_type": "rotary",
-    "rotary_scale": 1,
-    "rotary_dim": 128,
-    "rotary_base": 10000,
-    "rotary_adjacent_pairs": false,
-    "use_NTK_by_parts_rope": false,
-    "NTK_by_parts_low_freq_factor": null,
-    "NTK_by_parts_high_freq_factor": null,
-    "NTK_by_parts_factor": null,
-    "old_context_len": null
-}

L9Lorsa/final.pth

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ff7c8c57c2a245adec188f711e33c393da6f786e92908155b91ec059d65dec62
-size 57006482

L9Lorsa/metrics.json

@@ -1,11 +0,0 @@
-{
-    "mse_loss": 0.00258,
-    "explained_variance": 0.87377,
-    "l1": 18.61,
-    "ground_truth_norm": 4.069,
-    "reconstructed_norm": 3.853,
-    "error_norm": 1.223,
-    "sparsity/below 1e-5": 3197,
-    "sparsity/below 1e-6": 3019,
-    "positivity": 1.0
-}