qwen_new_mage_per_domain_balanced_moe_c10

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,90 @@
+---
+library_name: transformers
+license: other
+base_model: Qwen/Qwen1.5-MoE-A2.7B
+tags:
+- generated_from_trainer
+metrics:
+- accuracy
+model-index:
+- name: fine_tuned_per_domain_balanced_moe_c10
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# fine_tuned_per_domain_balanced_moe_c10
+
+This model is a fine-tuned version of [Qwen/Qwen1.5-MoE-A2.7B](https://huggingface.co/Qwen/Qwen1.5-MoE-A2.7B) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.2149
+- Accuracy: 0.5374
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 2e-05
+- train_batch_size: 1
+- eval_batch_size: 1
+- seed: 42
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- num_epochs: 3
+
+### Training results
+
+| Training Loss | Epoch  | Step | Accuracy | Validation Loss |
+|:-------------:|:------:|:----:|:--------:|:---------------:|
+| 7.9537        | 0.0006 | 100  | 0.5384   | 4.2406          |
+| 2.7142        | 0.0013 | 200  | 0.5386   | 6.1312          |
+| 2.5969        | 0.0019 | 300  | 0.4651   | 1.0811          |
+| 3.6087        | 0.0025 | 400  | 0.4655   | 1.7135          |
+| 3.217         | 0.0032 | 500  | 0.5386   | 2.4567          |
+| 2.0844        | 0.0038 | 600  | 0.4614   | 3.8137          |
+| 3.0955        | 0.0044 | 700  | 0.5386   | 1.2668          |
+| 2.0157        | 0.0051 | 800  | 0.5386   | 3.2796          |
+| 2.4513        | 0.0057 | 900  | 0.4614   | 2.2765          |
+| 2.482         | 0.0063 | 1000 | 0.5386   | 0.7492          |
+| 2.3079        | 0.0070 | 1100 | 0.5386   | 1.6933          |
+| 2.5698        | 0.0076 | 1200 | 0.5386   | 3.1721          |
+| 2.4214        | 0.0082 | 1300 | 0.5386   | 1.7702          |
+| 1.2708        | 0.0089 | 1400 | 0.4646   | 0.9111          |
+| 0.8665        | 0.0095 | 1500 | 0.5494   | 0.6819          |
+| 1.7844        | 0.0101 | 1600 | 0.5386   | 1.7757          |
+| 2.9675        | 0.0108 | 1700 | 0.5386   | 2.7387          |
+| 2.7119        | 0.0114 | 1800 | 0.5386   | 2.6287          |
+| 2.526         | 0.0120 | 1900 | 0.5386   | 1.4967          |
+| 3.2745        | 0.0127 | 2000 | 0.4614   | 4.2874          |
+| 3.4052        | 0.0133 | 2100 | 1.0082   | 0.4624          |
+| 1.7179        | 0.0139 | 2200 | 1.6046   | 0.4666          |
+| 2.7225        | 0.0146 | 2300 | 3.3510   | 0.5376          |
+| 2.2919        | 0.0152 | 2400 | 3.3149   | 0.5376          |
+| 1.729         | 0.0158 | 2500 | 2.1687   | 0.5376          |
+| 2.5072        | 0.0165 | 2600 | 2.9068   | 0.5376          |
+| 1.9138        | 0.0171 | 2700 | 1.4200   | 0.4624          |
+| 1.4881        | 0.0177 | 2800 | 2.2129   | 0.4631          |
+| 2.031         | 0.0184 | 2900 | 2.2580   | 0.5370          |
+| 1.998         | 0.0190 | 3000 | 2.2149   | 0.5374          |
+
+
+### Framework versions
+
+- Transformers 4.49.0
+- Pytorch 2.6.0+cu126
+- Datasets 3.3.2
+- Tokenizers 0.21.0

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

config.json

@@ -0,0 +1,40 @@
+{
+  "_name_or_path": "Qwen/Qwen1.5-MoE-A2.7B",
+  "architectures": [
+    "Qwen2MoeForSequenceClassification"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "decoder_sparse_step": 1,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 5632,
+  "max_position_embeddings": 8192,
+  "max_window_layers": 21,
+  "mlp_only_layers": [],
+  "model_type": "qwen2_moe",
+  "moe_intermediate_size": 1408,
+  "norm_topk_prob": false,
+  "num_attention_heads": 16,
+  "num_experts": 60,
+  "num_experts_per_tok": 4,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 16,
+  "output_router_logits": false,
+  "pad_token_id": 151643,
+  "problem_type": "single_label_classification",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.001,
+  "shared_expert_intermediate_size": 5632,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.49.0",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

model-00001-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0949c0e69342fc6ec5dc56e2007d5dae8b1340c9f430042ad055b02b4b6c637
+size 4990221104

model-00002-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e84fd413ac1d660779e391e299133362f37fbad52d502809382f2e6f0db609a6
+size 4991306528

model-00003-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5104bf21ea5b55553d274a2ad602021443e414bfb38393cf3207fd285743608e
+size 4990298240

model-00004-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f78d83cca86a2b8e3743d5d4d327c23afd8eb1dcd72fa3a0cf7c6496a4ba6a4e
+size 4990757696

model-00005-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dd05f558cc20d02a02d957aab86727bb3e34b051cb7e88a194876043630e955b
+size 4991306600

model-00006-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ea88a0630f5feb5b95c1f8d3ab48fcfdf275716224cba136f03ea6d2d3670ec
+size 4991306936

model-00007-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1f02f8c5a1f06714f2be735f3ebd19988ad5bf43530db35731933189183298b
+size 4991306952

model-00008-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b1b41c5f0ef04b056b85f78d0b4e7c39125b27c89bf94483c262062d4e979d2
+size 4968238032

model-00009-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82efdd1128051870bed853bc2c9987f329ac3cfbe3b3cf23d3879bc5c9ca7bb9
+size 4989749864

model-00010-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:152f919550b80e212f29d1d86e75e9dae8ddcbdd4238c9d056593df6b0227379
+size 4991306928

model-00011-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5333aa18a28c7a1168df97e5553e8bb2b849ca39d645a2a596be64da28a96355
+size 4991306936

model-00012-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:450e4efb1138eb53ad0bf9765658bc2661718a385e7a776ee65a023f3ff08c23
+size 1141959976

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ae291beb14c5986404b36fdaabbcf3cb858121c6239ffbec6c1405cbafd9dc
+size 11418544

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35c6d1b4bcc10c81455becaa2571094d6ea215e4411e0d29db27556b7dd26dd9
+size 5304