See axolotl config

axolotl version: 0.8.0.dev0

# === Start-up Commands ===
# curl -LsSf https://astral.sh/uv/install.sh | sh
# export PATH="$HOME/.local/bin:$PATH"
# git clone https://github.com/axolotl-ai-cloud/axolotl
# cd axolotl
# git checkout d8b4027200de0fe60f4ae0a71272c1a8cb2888f7
# uv venv
# source .venv/bin/activate
# uv pip install packaging ninja setuptools huggingface_hub[cli,hf_transfer]
# uv pip install "cut-cross-entropy[transformers] @ git+https://github.com/apple/ml-cross-entropy.git"
# uv pip install apollo-torch
# uv pip install --no-build-isolation -e .[flash-attn,deepspeed]
# uv pip install git+https://github.com/huggingface/transformers.git
# export HF_HUB_ENABLE_HF_TRANSFER=1
# huggingface-cli login --token $hf_key && wandb login $wandb_key
# axolotl preprocess qwen21-creative.yml
# axolotl train qwen21-creative.yml
# 39-43
# 36-40
# 25-29
# = 25-29 and 36-43

# curl -LsSf https://astral.sh/uv/install.sh | sh && export PATH="$HOME/.local/bin:$PATH" && git clone https://github.com/axolotl-ai-cloud/axolotl && cd axolotl && uv venv && source .venv/bin/activate && uv pip install torch==2.5.1 packaging ninja setuptools huggingface_hub[cli,hf_transfer] && uv pip install apollo-torch && uv pip install --no-build-isolation -e .[flash-attn,deepspeed] && uv pip install git+https://github.com/huggingface/transformers.git && export HF_HUB_ENABLE_HF_TRANSFER=1 && cd .. && huggingface-cli login --token $hf_key && wandb login $wandb_key

# === Model Configuration ===
base_model: Columbidae/Qwen2.5-21B-Experimental-E2
load_in_8bit: false
load_in_4bit: true

# === HF Configuration === 
hub_model_id: ToastyPigeon/qwen-21b-creative-qlora
hub_strategy: "every_save"

# === Training Setup ===
num_epochs: 1
micro_batch_size: 1
gradient_accumulation_steps: 4
sequence_len: 4096
sample_packing: true
pad_to_sequence_len: true

# === Evaluation ===
val_set_size: 100
evals_per_epoch: 10
eval_table_size:
eval_max_new_tokens: 256
eval_sample_packing: true

# === LoRA Configuration ===
adapter: qlora
lora_model_dir:
lora_r: 32
lora_alpha: 64
lora_dropout: 0.5
lora_target_linear: 
lora_fan_in_fan_out:
lora_target_modules:
  - gate_proj
  - down_proj
  - up_proj
  - q_proj
  - v_proj
  - k_proj
  - o_proj

#lora_mlp_kernel: true
#lora_qkv_kernel: true
#lora_o_kernel: true

# === Hyperparameter Configuration ===
#optimizer: apollo_adamw
optimizer: paged_ademamix_8bit
# Apollo-mini configuration:
#optim_args: "proj=random,rank=1,scale=128.0,scale_type=tensor,update_proj_gap=200"
# Regular Apollo configuration:
# optim_args: 
#optim_target_modules: all_linear
learning_rate: 1e-5
lr_scheduler: cosine
weight_decay: 0.01
warmup_ratio: 0.05


# === Data Configuration ===
datasets:
  - path: ToastyPigeon/mixed-instruct
    type: chat_template
    split: train[:300]
    field_messages: conversations
    message_field_role: from
    message_field_content: value
  - path: ToastyPigeon/some-rp-v2-4k
    split: train[:1000]
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
  - path: ToastyPigeon/gutenberg-sft
    split: train[500:1000]
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
  - path: ToastyPigeon/fujin-filtered-instruct
    split: train[:500]
    type: chat_template
    field_messages: conversations
    message_field_role: from
    message_field_content: value
  - path: ToastyPigeon/ali-books
    type: completion
    field: text
  - path: ToastyPigeon/disco-chat
    type: completion
    field: text
  - path: ToastyPigeon/adventure-combined-no-slop-matches
    split: train[:300]
    type: completion
    field: text

dataset_prepared_path: last_run_prepared
chat_template: tokenizer_default
# Example custom template:
# chat_template: jinja
# chat_template_jinja: |
#   {{- bos_token }}{%- for message in messages %}
#   {%- if message['role'] == 'system' %}
#   {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}
#   {%- elif message['role'] == 'user' %}
#   {{- '[INST]' + message['content'] + '[/INST]' }}
#   {%- elif message['role'] == 'assistant' %}
#   {{- message['content'] + eos_token }}
#   {%- endif %}
#   {%- endfor %}

# === Plugins ===
plugins:
  - axolotl.integrations.liger.LigerPlugin
#  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin

# === Hardware Optimization ===
gradient_checkpointing: unsloth
gradient_checkpointing_kwargs:
  use_reentrant: false
liger_rope: true
liger_rms_norm: true
liger_glu_activation: true
liger_fused_linear_cross_entropy: true
unsloth_cross_entropy_loss: true
#cut_cross_entropy: true
# Only if using multiple GPUs:
#deepspeed: axolotl/deepspeed_configs/zero3_bf16.json

# === Wandb Tracking ===
wandb_project: Qwen-27
# wandb_entity: [WANDB_ENTITY]
# wandb_name: [WANDB_RUN_NAME]

# === Checkpointing ===
saves_per_epoch: 10
save_total_limit: 1

# === Advanced Settings ===
output_dir: ./ckpts
bf16: auto
flash_attention: true
train_on_inputs: false
group_by_length: false
save_safetensors: true
logging_steps: 1
gc_steps: 10
seed: 69

qwen-21b-creative-qlora

This model is a fine-tuned version of Columbidae/Qwen2.5-21B-Experimental-E2 on the ToastyPigeon/mixed-instruct, the ToastyPigeon/some-rp-v2-4k, the ToastyPigeon/gutenberg-sft, the ToastyPigeon/fujin-filtered-instruct, the ToastyPigeon/ali-books, the ToastyPigeon/disco-chat and the ToastyPigeon/adventure-combined-no-slop-matches datasets. It achieves the following results on the evaluation set:

• Loss: 2.2767

Model description

More information needed

Intended uses & limitations

More information needed

Training and evaluation data

More information needed

Training procedure

Training hyperparameters

The following hyperparameters were used during training:

• learning_rate: 1e-05
• train_batch_size: 1
• eval_batch_size: 1
• seed: 69
• distributed_type: multi-GPU
• num_devices: 4
• gradient_accumulation_steps: 4
• total_train_batch_size: 16
• total_eval_batch_size: 4
• optimizer: Use OptimizerNames.PAGED_ADEMAMIX_8BIT and the args are: No additional optimizer arguments
• lr_scheduler_type: cosine
• lr_scheduler_warmup_steps: 14
• num_epochs: 1.0

Training results

Training Loss	Epoch	Step	Validation Loss
2.4726	0.0035	1	2.3354
2.382	0.1026	29	2.3293
2.4123	0.2051	58	2.3161
2.3212	0.3077	87	2.3042
2.4182	0.4103	116	2.2946
2.2086	0.5128	145	2.2878
2.2599	0.6154	174	2.2819
2.3582	0.7179	203	2.2789
2.3901	0.8205	232	2.2773
2.27	0.9231	261	2.2767

Framework versions

• PEFT 0.14.0
• Transformers 4.50.0.dev0
• Pytorch 2.5.1+cu124
• Datasets 3.2.0
• Tokenizers 0.21.0