--- library_name: peft base_model: Columbidae/Qwen2.5-21B-Experimental-E2 tags: - axolotl - generated_from_trainer datasets: - ToastyPigeon/mixed-instruct - ToastyPigeon/some-rp-v2-4k - ToastyPigeon/gutenberg-sft - ToastyPigeon/fujin-filtered-instruct - ToastyPigeon/ali-books - ToastyPigeon/disco-chat - ToastyPigeon/adventure-combined-no-slop-matches model-index: - name: qwen-21b-creative-qlora results: [] --- [Built with Axolotl](https://github.com/axolotl-ai-cloud/axolotl)
See axolotl config axolotl version: `0.8.0.dev0` ```yaml # === Start-up Commands === # curl -LsSf https://astral.sh/uv/install.sh | sh # export PATH="$HOME/.local/bin:$PATH" # git clone https://github.com/axolotl-ai-cloud/axolotl # cd axolotl # git checkout d8b4027200de0fe60f4ae0a71272c1a8cb2888f7 # uv venv # source .venv/bin/activate # uv pip install packaging ninja setuptools huggingface_hub[cli,hf_transfer] # uv pip install "cut-cross-entropy[transformers] @ git+https://github.com/apple/ml-cross-entropy.git" # uv pip install apollo-torch # uv pip install --no-build-isolation -e .[flash-attn,deepspeed] # uv pip install git+https://github.com/huggingface/transformers.git # export HF_HUB_ENABLE_HF_TRANSFER=1 # huggingface-cli login --token $hf_key && wandb login $wandb_key # axolotl preprocess qwen21-creative.yml # axolotl train qwen21-creative.yml # 39-43 # 36-40 # 25-29 # = 25-29 and 36-43 # curl -LsSf https://astral.sh/uv/install.sh | sh && export PATH="$HOME/.local/bin:$PATH" && git clone https://github.com/axolotl-ai-cloud/axolotl && cd axolotl && uv venv && source .venv/bin/activate && uv pip install torch==2.5.1 packaging ninja setuptools huggingface_hub[cli,hf_transfer] && uv pip install apollo-torch && uv pip install --no-build-isolation -e .[flash-attn,deepspeed] && uv pip install git+https://github.com/huggingface/transformers.git && export HF_HUB_ENABLE_HF_TRANSFER=1 && cd .. && huggingface-cli login --token $hf_key && wandb login $wandb_key # === Model Configuration === base_model: Columbidae/Qwen2.5-21B-Experimental-E2 load_in_8bit: false load_in_4bit: true # === HF Configuration === hub_model_id: ToastyPigeon/qwen-21b-creative-qlora hub_strategy: "every_save" # === Training Setup === num_epochs: 1 micro_batch_size: 1 gradient_accumulation_steps: 4 sequence_len: 4096 sample_packing: true pad_to_sequence_len: true # === Evaluation === val_set_size: 100 evals_per_epoch: 10 eval_table_size: eval_max_new_tokens: 256 eval_sample_packing: true # === LoRA Configuration === adapter: qlora lora_model_dir: lora_r: 32 lora_alpha: 64 lora_dropout: 0.5 lora_target_linear: lora_fan_in_fan_out: lora_target_modules: - gate_proj - down_proj - up_proj - q_proj - v_proj - k_proj - o_proj #lora_mlp_kernel: true #lora_qkv_kernel: true #lora_o_kernel: true # === Hyperparameter Configuration === #optimizer: apollo_adamw optimizer: paged_ademamix_8bit # Apollo-mini configuration: #optim_args: "proj=random,rank=1,scale=128.0,scale_type=tensor,update_proj_gap=200" # Regular Apollo configuration: # optim_args: #optim_target_modules: all_linear learning_rate: 1e-5 lr_scheduler: cosine weight_decay: 0.01 warmup_ratio: 0.05 # === Data Configuration === datasets: - path: ToastyPigeon/mixed-instruct type: chat_template split: train[:300] field_messages: conversations message_field_role: from message_field_content: value - path: ToastyPigeon/some-rp-v2-4k split: train[:1000] type: chat_template field_messages: conversations message_field_role: from message_field_content: value - path: ToastyPigeon/gutenberg-sft split: train[500:1000] type: chat_template field_messages: conversations message_field_role: from message_field_content: value - path: ToastyPigeon/fujin-filtered-instruct split: train[:500] type: chat_template field_messages: conversations message_field_role: from message_field_content: value - path: ToastyPigeon/ali-books type: completion field: text - path: ToastyPigeon/disco-chat type: completion field: text - path: ToastyPigeon/adventure-combined-no-slop-matches split: train[:300] type: completion field: text dataset_prepared_path: last_run_prepared chat_template: tokenizer_default # Example custom template: # chat_template: jinja # chat_template_jinja: | # {{- bos_token }}{%- for message in messages %} # {%- if message['role'] == 'system' %} # {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }} # {%- elif message['role'] == 'user' %} # {{- '[INST]' + message['content'] + '[/INST]' }} # {%- elif message['role'] == 'assistant' %} # {{- message['content'] + eos_token }} # {%- endif %} # {%- endfor %} # === Plugins === plugins: - axolotl.integrations.liger.LigerPlugin # - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin # === Hardware Optimization === gradient_checkpointing: unsloth gradient_checkpointing_kwargs: use_reentrant: false liger_rope: true liger_rms_norm: true liger_glu_activation: true liger_fused_linear_cross_entropy: true unsloth_cross_entropy_loss: true #cut_cross_entropy: true # Only if using multiple GPUs: #deepspeed: axolotl/deepspeed_configs/zero3_bf16.json # === Wandb Tracking === wandb_project: Qwen-27 # wandb_entity: [WANDB_ENTITY] # wandb_name: [WANDB_RUN_NAME] # === Checkpointing === saves_per_epoch: 10 save_total_limit: 1 # === Advanced Settings === output_dir: ./ckpts bf16: auto flash_attention: true train_on_inputs: false group_by_length: false save_safetensors: true logging_steps: 1 gc_steps: 10 seed: 69 ```

# qwen-21b-creative-qlora This model is a fine-tuned version of [Columbidae/Qwen2.5-21B-Experimental-E2](https://huggingface.co/Columbidae/Qwen2.5-21B-Experimental-E2) on the ToastyPigeon/mixed-instruct, the ToastyPigeon/some-rp-v2-4k, the ToastyPigeon/gutenberg-sft, the ToastyPigeon/fujin-filtered-instruct, the ToastyPigeon/ali-books, the ToastyPigeon/disco-chat and the ToastyPigeon/adventure-combined-no-slop-matches datasets. It achieves the following results on the evaluation set: - Loss: 2.2767 ## Model description More information needed ## Intended uses & limitations More information needed ## Training and evaluation data More information needed ## Training procedure ### Training hyperparameters The following hyperparameters were used during training: - learning_rate: 1e-05 - train_batch_size: 1 - eval_batch_size: 1 - seed: 69 - distributed_type: multi-GPU - num_devices: 4 - gradient_accumulation_steps: 4 - total_train_batch_size: 16 - total_eval_batch_size: 4 - optimizer: Use OptimizerNames.PAGED_ADEMAMIX_8BIT and the args are: No additional optimizer arguments - lr_scheduler_type: cosine - lr_scheduler_warmup_steps: 14 - num_epochs: 1.0 ### Training results | Training Loss | Epoch | Step | Validation Loss | |:-------------:|:------:|:----:|:---------------:| | 2.4726 | 0.0035 | 1 | 2.3354 | | 2.382 | 0.1026 | 29 | 2.3293 | | 2.4123 | 0.2051 | 58 | 2.3161 | | 2.3212 | 0.3077 | 87 | 2.3042 | | 2.4182 | 0.4103 | 116 | 2.2946 | | 2.2086 | 0.5128 | 145 | 2.2878 | | 2.2599 | 0.6154 | 174 | 2.2819 | | 2.3582 | 0.7179 | 203 | 2.2789 | | 2.3901 | 0.8205 | 232 | 2.2773 | | 2.27 | 0.9231 | 261 | 2.2767 | ### Framework versions - PEFT 0.14.0 - Transformers 4.50.0.dev0 - Pytorch 2.5.1+cu124 - Datasets 3.2.0 - Tokenizers 0.21.0