Upload 15 files

README.md

@@ -0,0 +1,26 @@
+---
+base_model:
+- jukofyork/Dark-Miqu-70B
+- sophosympatheia/Midnight-Miqu-70B-v1.5
+- jukofyork/Dawn-Miqu-70B
+library_name: transformers
+tags:
+- mergekit
+- merge
+license: other
+---
+Twilight Miqu is a Story writing model and is composed from sophosympatheia/Midnight-Miqu-70B-v1.5, jukofyork/Dawn-Miqu-70B and jukofyork/Dark-Miqu-70B
+
+It is an experiment to see if large models are more coherent on story writing tasks.
+Twilight = Midnight + Dawn + Dark 
+
+Please see this model card for details and usage instructions.
+https://huggingface.co/sophosympatheia/Midnight-Miqu-70B-v1.5
+
+This model is based on Miqu so it's capable of 32K context.
+
+All miqu-derived models, including this merge, are only suitable for personal use. Mistral has been cool about it so far, but you should be aware that by downloading this merge you are assuming whatever legal risk is inherent in acquiring and using a model based on leaked weights. This merge comes with no warranties or guarantees of any kind, but you probably already knew that.
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+A big thank you to Mistral, sophosympatheia and jukofyork for the original models!

config.json

@@ -0,0 +1,41 @@
+{
+    "_name_or_path": "softwareweaver/Twilight-Miqu-146B",
+    "architectures": [
+        "LlamaForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 8192,
+    "initializer_range": 0.02,
+    "intermediate_size": 28672,
+    "max_position_embeddings": 32764,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "num_attention_heads": 64,
+    "num_hidden_layers": 170,
+    "num_key_value_heads": 8,
+    "pad_token_id": 0,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": null,
+    "rope_theta": 1000000,
+    "tie_word_embeddings": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.41.1",
+    "use_cache": true,
+    "vocab_size": 32000,
+    "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.1.1",
+        "bits": 3.25,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
+}

mergekit_config.yml

@@ -0,0 +1,88 @@
+const_tag: &MODEL1 sophosympatheia/Midnight-Miqu-70B-v1.5
+const_tag: &MODEL3 jukofyork/Dawn-Miqu-70B
+const_tag: &MODEL2 jukofyork/Dark-Miqu-70B
+
+const_tag: &QK_ATTENUATION_FACTOR 0.8408964153  # sqrt(sqrt(1/2))
+const_tag: &MLP_DOWN_SCALE_FACTOR 0.7071067812  # sqrt(1/2)
+
+scale-filter-env: &scale_filter_env
+  parameters:
+    scale:
+      - filter: q_proj
+        value: *QK_ATTENUATION_FACTOR
+      - filter: k_proj
+        value: *QK_ATTENUATION_FACTOR
+      - filter: down_proj
+        value: *MLP_DOWN_SCALE_FACTOR
+      - value: 1.0
+
+slices:
+  - sources:
+    - model: *MODEL1
+      layer_range: [0, 10]
+  - sources:
+    - model: *MODEL1
+      layer_range: [10, 20]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL2
+      layer_range: [10, 20]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL3
+      layer_range: [10, 20]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL3
+      layer_range: [20, 30]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL2
+      layer_range: [20, 30]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL1
+      layer_range: [30, 40]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL2
+      layer_range: [30, 40]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL3
+      layer_range: [40, 50]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL2
+      layer_range: [40, 50]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL1
+      layer_range: [50, 60]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL2
+      layer_range: [50, 60]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL3
+      layer_range: [50, 60]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL1
+      layer_range: [60, 70]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL2
+      layer_range: [60, 70]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL3
+      layer_range: [60, 70]
+      <<: *scale_filter_env
+  - sources:
+    - model: *MODEL1
+      layer_range: [70, 80]
+
+merge_method: passthrough
+dtype: float16

output-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:910f95d089734d051fd93fbda610f86125e0a14fa38b5cf863176a987a2d8d96
+size 8569296702

output-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0722fa03598d916896ee466bc9152f6372211395daa3fc5e142603e822d0c77
+size 8534818552

output-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:826051cb9bc1a2521e56b59c6913be453e013b2771ac6cde726dd4a005b4d5da
+size 8555759082

output-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1eb82878a8f062934f87eccc567418304d75ac10dfacc407c8c259b0087f97fc
+size 8571517546

output-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52db074d34e2165b47b602565c00bf3817d35c3935f4f16464bdb6c4188c2919
+size 8547881050

output-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f342830b0a61e6043246fd8238b3a6e5ba9819b5f757d167cd56d14b8e41d19
+size 8547110166

output-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3506a9e81b15dd308b3ef954bdf14879d38076f2a34be3f8d7ccc8e395dce8dc
+size 8500164638

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}