Hugging Face's logo

sohampnow
/
olmoe_l20_h3072_ffn2048_4x32

Safetensors
olmoe
Model card Files Community
sohampnow commited on
Commit
89d9a11
·
verified ·
1 Parent(s): 825a5da

Upload folder using huggingface_hub

Browse files
Files changed (20) hide show
  1. .gitattributes +1 -0
  2. config.json +33 -0
  3. generation_config.json +4 -0
  4. generation_config_bak.json +6 -0
  5. metadata.json +1 -0
  6. model-00001-of-00011.safetensors +3 -0
  7. model-00002-of-00011.safetensors +3 -0
  8. model-00003-of-00011.safetensors +3 -0
  9. model-00004-of-00011.safetensors +3 -0
  10. model-00005-of-00011.safetensors +3 -0
  11. model-00006-of-00011.safetensors +3 -0
  12. model-00007-of-00011.safetensors +3 -0
  13. model-00008-of-00011.safetensors +3 -0
  14. model-00009-of-00011.safetensors +3 -0
  15. model-00010-of-00011.safetensors +3 -0
  16. model-00011-of-00011.safetensors +3 -0
  17. model.safetensors.index.json +0 -0
  18. special_tokens_map.json +23 -0
  19. tokenizer.json +3 -0
  20. tokenizer_config.json +0 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "allenai/OLMoE-1B-7B-0924",
3
+ "architectures": [
4
+ "OlmoeForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "clip_qkv": null,
9
+ "eos_token_id": 50279,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 3072,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 2048,
14
+ "max_position_embeddings": 4096,
15
+ "model_type": "olmoe",
16
+ "norm_topk_prob": false,
17
+ "num_attention_heads": 16,
18
+ "num_experts": 32,
19
+ "num_experts_per_tok": 4,
20
+ "num_hidden_layers": 20,
21
+ "num_key_value_heads": 16,
22
+ "output_router_logits": false,
23
+ "pad_token_id": 1,
24
+ "rms_norm_eps": 1e-05,
25
+ "rope_scaling": null,
26
+ "rope_theta": 10000.0,
27
+ "router_aux_loss_coef": 0.01,
28
+ "tie_word_embeddings": false,
29
+ "torch_dtype": "float32",
30
+ "transformers_version": "4.48.2",
31
+ "use_cache": true,
32
+ "vocab_size": 131072
33
+ }
generation_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "transformers_version": "4.48.2"
4
+ }
generation_config_bak.json ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "eos_token_id": 50279,
4
+ "pad_token_id": 1,
5
+ "transformers_version": "4.48.2"
6
+ }
metadata.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"num_active_params": "3.07B", "total_params": "13.64B"}
model-00001-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:af5fed4ac32d36a9a0bded7307f7dd6187f4b54fe1eadd14479165a4c614cdbf
3
+ size 4983710256
model-00002-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8c239c9e140d5cd364eb86c5a5c8bcbd323b213c25f5210c8f8bc0e05953c94
3
+ size 4983742760
model-00003-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2cde90abfe39dcaa6fa7f166c74a0f5a54624875bf2ffb1a5879fca6a3f69dba
3
+ size 4983742760
model-00004-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:97ad8b3eb12d12ddfe10811dfc1ecb8f98519d0a1d3d439247acc567e4e0eea9
3
+ size 4983742760
model-00005-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ec3891cac2a84f03e0162c2fae14d28e016914f35a1bcef01211cd29495cd9f
3
+ size 4983742760
model-00006-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0204bb0ea869cc898461832bc2bf5303bade8115a7139809a7799339e9ed3f35
3
+ size 4970741976
model-00007-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebcb63845ecc5c87084c37d339a0b4758d5ef5d164fb6283332db77cb352aa8d
3
+ size 4996301192
model-00008-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f5b2c215359e882b0be78fb22643793bf6d21786f032344d21f569a27db0949
3
+ size 4983742952
model-00009-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:773ed0b384cf57988cd0ea12d9459a508dc08d6de6ae6a7613e9562f5f664bbd
3
+ size 4983742952
model-00010-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:88ef32d6ca88581bc72ec95ded1ea735e32cfd370e8245bc5c8f2a9421584476
3
+ size 4983742952
model-00011-of-00011.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68794ec2b29644751f9051f580ba3965da0823be3d1bd9a49d9a69894f7d44cf
3
+ size 4731670264
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "unk_token": {
17
+ "content": "<unk>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b0240ce510f08e6c2041724e9043e33be9d251d1e4a4d94eb68cd47b954b61d2
3
+ size 17078292
tokenizer_config.json ADDED
The diff for this file is too large to render. See raw diff