Upload folder using huggingface_hub

README.md

@@ -0,0 +1,40 @@
+---
+base_model:
+- alpindale/WizardLM-2-8x22B
+- HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# merge
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the SLERP merge method.
+
+### Models Merged
+
+The following models were included in the merge:
+* [alpindale/WizardLM-2-8x22B](https://huggingface.co/alpindale/WizardLM-2-8x22B)
+* [HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1)
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+models:
+  - model: alpindale/WizardLM-2-8x22B
+  - model: HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1
+merge_method: slerp
+base_model: alpindale/WizardLM-2-8x22B
+dtype: bfloat16
+parameters:
+  t: [0, 0.5, 1, 0.5, 0] # V shaped curve: Hermes for input & output, WizardMath in the middle layers
+
+```

config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "alpindale/WizardLM-2-8x22B",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 6144,
+  "initializer_range": 0.02,
+  "intermediate_size": 16384,
+  "max_position_embeddings": 65536,
+  "model_type": "mixtral",
+  "num_attention_heads": 48,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}

mergekit_config.yml

@@ -0,0 +1,8 @@
+models:
+  - model: alpindale/WizardLM-2-8x22B
+  - model: HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1
+merge_method: slerp
+base_model: alpindale/WizardLM-2-8x22B
+dtype: bfloat16
+parameters:
+  t: [0, 0.5, 1, 0.5, 0] # V shaped curve: Hermes for input & output, WizardMath in the middle layers

model-00001-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:609a9fe55d983ffea23eef73099784f9f47262a22a778724874ad134b0c3778d
+size 9981722448

model-00002-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b9fde48abeafa17d6d65d74ebcf4e5c43c05a8f535e2ddb0383c403649e1573
+size 9814924936

model-00003-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e09123cb5389ff7d8195c018ebda18d43d4ac85cdaf58622d084ced1c7f27d31
+size 9814924936

model-00004-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abd3763b6e2a43292d39fb0382ec0aad65faa24dbd93422dd29181012e637c7b
+size 9814924936

model-00005-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a098a6c7c1f8d74f259dd02bb5dc8fce9b700c60b7b57137659403287a4bf229
+size 9814924936

model-00006-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:820539958a4dffcbe675d014a8d0d5c72a7d64dc78180ec7f6a08ce3d2d4a2e8
+size 9814924984

model-00007-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83f069fea192d47694b8cc09ca96f7540b2560491fe2cacbc2e0a5a1b611f601
+size 9814925000

model-00008-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0060072e7800d397b0787c5b6817beabca3c539a92b661b675c40c3571048d6
+size 9814949808

model-00009-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cba270b172f798ca49c397ff1350845e453f4a6c41503c17c1b85fb9a6b7f4d
+size 9814925000

model-00010-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9be27fd5cd646d3d00359e0c8e1a7856fac6a6194e7a447a05daa7d480e95cae
+size 9814925000

model-00011-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c919cf2a5c09ec01ab85e3569c008e435f5e536f3024d2babdc2dda6b534e07d
+size 9814925000

model-00012-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:385c75195cf5b0ac85abda65a70b3b3921eb5b3ceb11769ae62d6996f6a88ce3
+size 9928072680

model-00013-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89e580d30905bfc0f440aeaa8f77e73b2ce7ce35f14c1bacccaaa17ce14c72e7
+size 9928171104

model-00014-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0866c45fd2da10a27612aa035c42144a3f4af3f161b2f849ab75385263263457
+size 9814875384

model-00015-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ad254957c092ab06ba95f75f0c73dcfbd067b4a4750fede03996d809bac32a3
+size 9814925000

model-00016-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c7184dc03a9719fb0496cf679df68b8db8aa6e541806a01f4cf863f88f0c05d
+size 9814925000

model-00017-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1edb7bc4aea3720b1b983261d3944ed3f160e411f61c210ae5c05ba8239d79ec
+size 9814925000

model-00018-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:637edc3a335f1d625e868ecec5f8e157a790c54923d838d53bb64707ea99d4df
+size 9814949808

model-00019-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3669e0b08b7d9a8416c3ddba18dd9f0d0ee48e18b25e8724e36ad04cfa3b279
+size 9814925000

model-00020-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0c6dddabcc175dc2b79cc1e514262a0993a4225360cd69c607e14e0f5acdbed
+size 9814925000

model-00021-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d31f964189ff43bb1ab8f82ae2a55ac684f4ab8a9f941584eeaece2ff22fdf91
+size 9814925000

model-00022-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1974c7973c5633ef737361bb82c9dc5cfeb36712fa2f1c00e2d24bd3e65883ef
+size 9814925000

model-00023-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a273e5b05d6c604faf6dd37831ce166bd13a70a06de64bf60ba7dae964d6d55a
+size 9814925000

model-00024-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9c161a210d3e60f3176e0a17252b98ead87de6d710238b8c295331120f2f6f44
+size 9814925000

model-00025-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0dcaac8df53c3f18417c3d6639cd16ad7017e40213bc0b25968caac0110da5f3
+size 9814925000

model-00026-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:839abbcd126a49a0404830f22875b680eeabb20dcb351ad248b6270ad2eb8d77
+size 9928072680

model-00027-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:737498e0e81a9295ea39a58be33c459cded3cdc9695810c1411fa1ae4bcbfce6
+size 9928171104

model-00028-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51ac4eaa5ba47e2e636c9f83308e77d5232fc8bb2c587c178952868dbd70cbce
+size 9991159856

model-00029-of-00029.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:119f2ecc6156136332fc230ed88faf53a9feb13376b6212e043209488b4a2231
+size 5627772936

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,45 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{{ messages[0]['content'].strip() }}{% else %}{% set loop_messages = messages %}{{ 'A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user\\'s questions.' }}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ ' USER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% else %}{% if message['role'] == 'system' or message['role'] == 'user' %}{{ '\nUSER: ' + message['content'].strip() }}{% else %}{{ ' ASSISTANT: ' + message['content'].strip() + eos_token }}{% endif %}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ ' ASSISTANT:' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true
+}