Hugging Face's logo

RedDragonGecko
/
Progeny_2x70b

Safetensors
mixtral
Model card Files Community
RedDragonGecko commited on
Commit
7c53598
·
verified ·
1 Parent(s): 2463388

Upload 58 files

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. config.json +46 -0
  3. mergekit_moe_config.yml +6 -0
  4. model-00001-of-00052.safetensors +3 -0
  5. model-00002-of-00052.safetensors +3 -0
  6. model-00003-of-00052.safetensors +3 -0
  7. model-00004-of-00052.safetensors +3 -0
  8. model-00005-of-00052.safetensors +3 -0
  9. model-00006-of-00052.safetensors +3 -0
  10. model-00007-of-00052.safetensors +3 -0
  11. model-00008-of-00052.safetensors +3 -0
  12. model-00009-of-00052.safetensors +3 -0
  13. model-00010-of-00052.safetensors +3 -0
  14. model-00011-of-00052.safetensors +3 -0
  15. model-00012-of-00052.safetensors +3 -0
  16. model-00013-of-00052.safetensors +3 -0
  17. model-00014-of-00052.safetensors +3 -0
  18. model-00015-of-00052.safetensors +3 -0
  19. model-00016-of-00052.safetensors +3 -0
  20. model-00017-of-00052.safetensors +3 -0
  21. model-00018-of-00052.safetensors +3 -0
  22. model-00019-of-00052.safetensors +3 -0
  23. model-00020-of-00052.safetensors +3 -0
  24. model-00021-of-00052.safetensors +3 -0
  25. model-00022-of-00052.safetensors +3 -0
  26. model-00023-of-00052.safetensors +3 -0
  27. model-00024-of-00052.safetensors +3 -0
  28. model-00025-of-00052.safetensors +3 -0
  29. model-00026-of-00052.safetensors +3 -0
  30. model-00027-of-00052.safetensors +3 -0
  31. model-00028-of-00052.safetensors +3 -0
  32. model-00029-of-00052.safetensors +3 -0
  33. model-00030-of-00052.safetensors +3 -0
  34. model-00031-of-00052.safetensors +3 -0
  35. model-00032-of-00052.safetensors +3 -0
  36. model-00033-of-00052.safetensors +3 -0
  37. model-00034-of-00052.safetensors +3 -0
  38. model-00035-of-00052.safetensors +3 -0
  39. model-00036-of-00052.safetensors +3 -0
  40. model-00037-of-00052.safetensors +3 -0
  41. model-00038-of-00052.safetensors +3 -0
  42. model-00039-of-00052.safetensors +3 -0
  43. model-00040-of-00052.safetensors +3 -0
  44. model-00041-of-00052.safetensors +3 -0
  45. model-00042-of-00052.safetensors +3 -0
  46. model-00043-of-00052.safetensors +3 -0
  47. model-00044-of-00052.safetensors +3 -0
  48. model-00045-of-00052.safetensors +3 -0
  49. model-00046-of-00052.safetensors +3 -0
  50. model-00047-of-00052.safetensors +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Tarek07/Primogenitor-V2-LLaMa-70B",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 128000,
9
+ "eos_token_id": [
10
+ 128001,
11
+ 128008,
12
+ 128009
13
+ ],
14
+ "head_dim": 128,
15
+ "hidden_act": "silu",
16
+ "hidden_size": 8192,
17
+ "initializer_range": 0.02,
18
+ "intermediate_size": 28672,
19
+ "max_position_embeddings": 131072,
20
+ "mlp_bias": false,
21
+ "model_type": "mixtral",
22
+ "num_attention_heads": 64,
23
+ "num_experts_per_tok": 2,
24
+ "num_hidden_layers": 80,
25
+ "num_key_value_heads": 8,
26
+ "num_local_experts": 2,
27
+ "output_router_logits": false,
28
+ "pretraining_tp": 1,
29
+ "rms_norm_eps": 1e-05,
30
+ "rope_scaling": {
31
+ "factor": 8.0,
32
+ "high_freq_factor": 4.0,
33
+ "low_freq_factor": 1.0,
34
+ "original_max_position_embeddings": 8192,
35
+ "rope_type": "llama3"
36
+ },
37
+ "rope_theta": 500000.0,
38
+ "router_aux_loss_coef": 0.001,
39
+ "router_jitter_noise": 0.0,
40
+ "sliding_window": null,
41
+ "tie_word_embeddings": false,
42
+ "torch_dtype": "bfloat16",
43
+ "transformers_version": "4.49.0",
44
+ "use_cache": true,
45
+ "vocab_size": 128256
46
+ }
mergekit_moe_config.yml ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ base_model: Tarek07/Primogenitor-V2-LLaMa-70B
2
+ gate_mode: random
3
+ dtype: bfloat16
4
+ experts:
5
+ - source_model: Tarek07/Primogenitor-V2-LLaMa-70B
6
+ - source_model: Steelskull/L3.3-MS-Nevoria-70b
model-00001-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:df88fe829389203a4aa36f2e7dbee3e97f52deb8eab848624cc357afd1d2de3f
3
+ size 4752180696
model-00002-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d409ffad397dd24f6994da02dde419519c558904d91164dcae348af8a7783a92
3
+ size 4831906352
model-00003-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ed352454160aea0f69e6322b30bdd34fdf6b28054b87a757d80a0fb5032a880a
3
+ size 4999661672
model-00004-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:751d65236fc208e977628fdc340acd814e8f29dd52de0c924ceab0e036aac1e3
3
+ size 4831889856
model-00005-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9239888c9c8c45effc7f5be964a3f876bfc05637bcd1b0e0911d6b0b543afb35
3
+ size 4831906352
model-00006-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a0b2e8797461f89b83d112d6a6d03632fd566432227fef0e7f0cf4964fddd292
3
+ size 4999645176
model-00007-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a14156415cd61c6c61c628c2cf2826757c2625fb9b3061f28fe5dc21940a9be7
3
+ size 4831906352
model-00008-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bf90542aab32fa5277273b8ae932681c51e911533c4d99bf1a9f37bd06b6c5c1
3
+ size 4999645192
model-00009-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9c060dbb5203d406483dfac6166983792480fd0b32da6aed01a79d4d0464dd9
3
+ size 4831906376
model-00010-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b6ff446a0978ae95c32447aec123d15179b2e8374480a4868261174869a72ab9
3
+ size 4999661688
model-00011-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:29e349fe4d073d2bd35af5766377d8870bcb41723e54e11e4e84ade02c4f4ecd
3
+ size 4831889880
model-00012-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:144054face0a9015bd4968428057be0a3ea0a850605d9eb7e616069cf7060524
3
+ size 4831906376
model-00013-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5b130f52e08883e003f8e93b554fbc2a7a8ea8865c5bbc2b6b4ce111acaa159
3
+ size 4999645192
model-00014-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e939b52244be3fd63d9aac8245cce51661cc62c157d0c1b0461f9ece94d2a849
3
+ size 4831906376
model-00015-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:98387c250af40cedb42a22ba0a4ea2e357224ef1441cb2c27896285b141f23e1
3
+ size 4999645192
model-00016-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0d8dad069162826129ed6dbbf3f9829fa90ea0aeb9522eddf7d7d683aad17525
3
+ size 4831906376
model-00017-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7a58baabfaf67c86ed88f0f243255e71588a172060645fc3a50f303ef2b96452
3
+ size 4999661688
model-00018-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ea8d303d5d17b91943ad33b28e6f670ea21818fa9a4a012423fd3b0044e22667
3
+ size 4831889880
model-00019-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7a3ccd9de444bf38ae2d94dedb81fa3e751eaaca313c9e3b9aa62347e533591
3
+ size 4831906376
model-00020-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c812e8dcf2cf26505b0a37f26839d5ff66d88487fb27321b28b0832458bf0fb5
3
+ size 4999645192
model-00021-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:06d7e4db2fb5deb1e5a9c3aa5004eae1d50196bfd637d8f964be5140dbb7902e
3
+ size 4831906376
model-00022-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:16d1eec80078e0c65488fb66f46bf941d967a4a8a0bf727e11bf7c6d51bd3d79
3
+ size 4999645192
model-00023-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c766bc1f5eb4c4d41948d5abeb281a0960028d52f197806ae2d9424db959e893
3
+ size 4831906376
model-00024-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6257843f34ebce258db34d79e9318d41fb146f6e476c40728901e0cc0761a345
3
+ size 4999661688
model-00025-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eb08750eaa3281aab2844eda4478ac3d809d85ed903d12321a3ebf73c99ba66d
3
+ size 4831889880
model-00026-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:282863f862230b8f0dcc893ed83d6aaf34a5081c6d03cf9f98dcff4afc462b9f
3
+ size 4831906376
model-00027-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:baf885b2c260a29afee48b593fc3162db0e903261f01a80c7ba58e3ce2243794
3
+ size 4999645192
model-00028-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ca43b4f29006b6f93f24591a0a7bb6410ea39653ca41a59f55fe4023a827232
3
+ size 4831906376
model-00029-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3e8ddac601e31f36e24afcb1bd158ba620f5e1b63111dc8cfab78a7836f31292
3
+ size 4999645192
model-00030-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d7df682a6a90df5790d3bd087ce08dc88b11663f06c1a216b0a12bc11231feff
3
+ size 4831906376
model-00031-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eac214afc410167804424fbb4fdec241ab05c44c5bbd5d8c06fe9fdbba1b6e89
3
+ size 4999661688
model-00032-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:adc761230b9d1c8f652fb42188a5ad557ebf8182be36c2bab8e768256fe62b72
3
+ size 4831889880
model-00033-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6619864a2f702d7fdcab0500420937d0137e958966c3eae5d582ffedc9c00d32
3
+ size 4831906376
model-00034-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:89d13a670cfc1c294c2915f59c806385940ed14463a695b05c768eba08c1c5af
3
+ size 4999645192
model-00035-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e0ed52cfe44a90483718f24267ab9afa69b62a5ed7a7bcaa1b368e3d376a17f5
3
+ size 4831906376
model-00036-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:76b990218c9ec46ce46137c38ca4c41c970099cc78ef4d9593278999f87ce4a4
3
+ size 4999645192
model-00037-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:692f70373ebc5fd3bfb5136db7c9536f40b40a67a74ff46c7c6562dae7f85ecc
3
+ size 4831906376
model-00038-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4e11bcfd3cb8746b2960d6f8b31c9aca56cba96cfd6ffe47456b0a8eb8c07cdb
3
+ size 4999661688
model-00039-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b801ca245bcbfa4c8c9c6c7698e8c761530669b018e28036092656f08922256c
3
+ size 4831889880
model-00040-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:078658ed78dd092a585cbd21a6f52f1ae9014063290cf22361221eb23d129bdc
3
+ size 4831906376
model-00041-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8312c69b1b92fb9924fbc793714818665c3cce6fd82c8dbff902277d81433bfd
3
+ size 4999645192
model-00042-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c7c98867a4ab5cbff96bdfba13104bbc6517de53a014316c476d99f1cb02a6b4
3
+ size 4831906376
model-00043-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bfc247d44d1a7e14ce4678a7c84fc0450252cd046b3f74813317756877fffc5d
3
+ size 4999645192
model-00044-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a64c790f8ade65e5c2973fd5c9c1dead983c4b601984c3c3a219ab99cc34db27
3
+ size 4831906376
model-00045-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f757e1ea6ef2eb4dc1df4c9b1f13433939b1726fd20a59bdb817d9458734c7a
3
+ size 4999661688
model-00046-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b268a13d1cc1f87617195660a84a53b200cd8bcc6f25ed80448b3f197fafe7ab
3
+ size 4831889880
model-00047-of-00052.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09053ad2e8efd8877e15eda2afeb9dc0f26a50363f569dc6955ae4141aec1886
3
+ size 4831906376