--- base_model: google/gemma-3-4b-it license: gemma pipeline_tag: text-generation library_name: peft language: - en datasets: - BeaverAI/REDACTED1 - BeaverAI/REDACTED2 - BeaverAI/REDACTED3 - BeaverAI/REDACTED4 - PJMixers-Dev/Lit-axo-Shuffled - PJMixers-Dev/Mielikki_Erebus-87k-axo - PJMixers/RyokoAI_Honeyfeed3600-Cleanish - PJMixers-Dev/allura-org_fujin-cleaned-stage-2-axo - Nelathan/synthetic-sugar-quill - PJMixers-Dev/winglian_visual-novels-json-axo - PJMixers-Dev/recursal_SCP-RECURSAL-Cleaned - PJMixers-Dev/Subtitles - PJMixers-Dev/KaraKaraWitch_AnimeSubtitle-axo - PJMixers-Dev/Fundus-105K-Formatted - PJMixers-Dev/Fundus-AP-News-Formatted - PJMixers/AP-News-2024 - PJMixers-Dev/goodwiki-2024-12-04-axo - epfl-llm/guidelines - PJMixers-Dev/Gryphe-Aesir-RPG-Charcards-Opus-Mixed - allura-org/gryphe-sonnet-3.5-charcards-names-added - anthracite-org/c2_logs_32k_llama3_qwen2_v1.3 - PJMixers-Dev/MinervaAI_Aesir-Preview-Anon - PJMixers-Dev/lemonilia_LimaRP-Simple-CustomShareGPT-Shuffled - Epiculous/SynthRP-Gens-v1.1-Filtered-n-Cleaned - PJMixers-Dev/NyxKrage_chub-logs-sharegpt-longest-CustomShareGPT - PJMixers/OpenLeecher_Teatime_all_logs_longest-ShareGPT - grimulkan/aicg-logs-augmented - grimulkan/PIPPA-augmented-dedup - PJMixers/grimulkan_bluemoon_Karen_cleaned-carded-formatted - PJMixers/lodrick-the-lafted_OpusStories-ShareGPT - Gryphe/ChatGPT-4o-Writing-Prompts - Gryphe/Opus-WritingPrompts - anthracite-org/nopm_claude_writing_fixed - PJMixers-Dev/Tiefighter-13B-Fake-Distill-ShareGPT - allura-org/fujin-instruct-v2 - PocketDoc/Dans-Prosemaxx-Adventure - PocketDoc/Dans-Failuremaxx-Adventure-3 --- # Gemma-3-Earthen-v0.2-4B-QLoRA [`google/gemma-3-4b-it`](https://huggingface.co/google/gemma-3-4b-it) was trained at 8K with batch size 4 gradient accumulation 4, so each step was 131,072 tokens (including any padding tokens). It was trained for 160 steps, adding up to a total of 20,971,520 unique tokens seen. This is a small test run. A larger version is planned. ## Quants - [GGUF from mradermacher](https://huggingface.co/mradermacher/Gemma-3-Earthen-v0.2-4B-GGUF) ## Prompt Format This model uses Gemma-3 Instruct format, but with system turn support. ## Training Details [Built with Axolotl](https://github.com/axolotl-ai-cloud/axolotl) ```yaml # Requirements before running # - Get latest commit of axolotl (currently c0a0c75) # - Download these to axolotl/src/axolotl/prompt_formatters # - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/formatter_regex.py # - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/customcompletion-regex.py # - https://github.com/xzuyn/axolotl/blob/came-plus-formatters/src/axolotl/prompt_strategies/customgemma3-regex.py # - pip install ftfy # - pip install git+https://github.com/xzuyn/CAME.git@sr-grams-cautious-8bit # Weights and Biases logging config wandb_project: Gemma-3-4B wandb_entity: wandb_watch: wandb_name: Gemma-3-Earthen-v0.2-4B-QLoRA-run1 wandb_log_model: # Model checkpointing config output_dir: ./Outputs/Gemma-3-Earthen-v0.2-4B-QLoRA-run1 save_steps: 10 save_safetensors: true save_total_limit: 2 save_only_model: true # Model architecture config base_model: google/gemma-3-4b-it model_type: AutoModelForCausalLM tokenizer_type: AutoTokenizer # Mixed precision training config bf16: true fp16: false tf32: false # Model loading config load_in_8bit: false load_in_4bit: true strict: false # Sequence config sequence_len: 8192 min_sample_len: 256 sample_packing: true eval_sample_packing: true pad_to_sequence_len: true train_on_inputs: false group_by_length: false # LoRA adapter config adapter: qlora lora_model_dir: lora_r: 256 lora_alpha: 256 lora_dropout: 0.125 lora_target_modules: 'language_model.model.layers.[\d]+.(mlp|cross_attn|self_attn).(up|down|gate|q|k|v|o)_proj' embeddings_skip_upcast: true # Dataset config datasets: # Completion # Story-like Data - path: BeaverAI/REDACTED1 split: train[:1000] type: customcompletion-regex - path: PJMixers-Dev/Lit-axo-Shuffled split: train[:1000] type: customcompletion-regex - path: PJMixers-Dev/Mielikki_Erebus-87k-axo split: train[:1000] type: customcompletion-regex - path: PJMixers/RyokoAI_Honeyfeed3600-Cleanish split: train[:1000] type: customcompletion-regex - path: BeaverAI/REDACTED2 split: train[:1000] type: customcompletion-regex - path: PJMixers-Dev/allura-org_fujin-cleaned-stage-2-axo split: train[:1000] type: customcompletion-regex - path: Nelathan/synthetic-sugar-quill split: train[:1000] type: customcompletion-regex - path: PJMixers-Dev/winglian_visual-novels-json-axo split: train[:1000] type: customcompletion-regex - path: BeaverAI/REDACTED3 split: train[:1000] type: customcompletion-regex - path: PJMixers-Dev/recursal_SCP-RECURSAL-Cleaned split: train[:1000] type: customcompletion-regex # Subtitle Data - path: PJMixers-Dev/Subtitles split: train[:1000] type: customcompletion-regex - path: PJMixers-Dev/KaraKaraWitch_AnimeSubtitle-axo split: train[:1000] type: customcompletion-regex # News Data - path: PJMixers-Dev/Fundus-105K-Formatted split: train[:1000] type: customcompletion-regex - path: PJMixers-Dev/Fundus-AP-News-Formatted split: train[:1000] type: customcompletion-regex - path: PJMixers/AP-News-2024 split: train[:1000] type: customcompletion-regex # Misc Data - path: PJMixers-Dev/goodwiki-2024-12-04-axo split: train[:1000] type: customcompletion-regex - path: epfl-llm/guidelines split: train[:1000] field: clean_text type: customcompletion-regex # Gemma-3 Instruct # RP Data - path: PJMixers-Dev/Gryphe-Aesir-RPG-Charcards-Opus-Mixed type: customgemma3-regex - path: allura-org/gryphe-sonnet-3.5-charcards-names-added type: customgemma3-regex - path: anthracite-org/c2_logs_32k_llama3_qwen2_v1.3 type: customgemma3-regex - path: BeaverAI/REDACTED4 type: customgemma3-regex - path: PJMixers-Dev/MinervaAI_Aesir-Preview-Anon type: customgemma3-regex - path: PJMixers-Dev/lemonilia_LimaRP-Simple-CustomShareGPT-Shuffled type: customgemma3-regex - path: Epiculous/SynthRP-Gens-v1.1-Filtered-n-Cleaned type: customgemma3-regex - path: PJMixers-Dev/NyxKrage_chub-logs-sharegpt-longest-CustomShareGPT type: customgemma3-regex - path: PJMixers/OpenLeecher_Teatime_all_logs_longest-ShareGPT type: customgemma3-regex - path: grimulkan/aicg-logs-augmented type: customgemma3-regex - path: grimulkan/PIPPA-augmented-dedup type: customgemma3-regex - path: PJMixers/grimulkan_bluemoon_Karen_cleaned-carded-formatted type: customgemma3-regex # InstStory Data - path: PJMixers/lodrick-the-lafted_OpusStories-ShareGPT type: customgemma3-regex - path: Gryphe/ChatGPT-4o-Writing-Prompts type: customgemma3-regex - path: Gryphe/Opus-WritingPrompts type: customgemma3-regex - path: anthracite-org/nopm_claude_writing_fixed type: customgemma3-regex - path: PJMixers-Dev/Tiefighter-13B-Fake-Distill-ShareGPT type: customgemma3-regex - path: allura-org/fujin-instruct-v2 type: customgemma3-regex # Adventure Data - path: PocketDoc/Dans-Prosemaxx-Adventure type: customgemma3-regex - path: PocketDoc/Dans-Failuremaxx-Adventure-3 type: customgemma3-regex test_datasets: val_set_size: 256 eval_strategy: steps eval_steps: 10 dataset_prepared_path: ./00-Tokenized-Datasets/Gemma-3-Earthen-v0.2-4B-LoRA-seed42 shuffle_merged_datasets: true dataset_processes: # Training hyperparameters num_epochs: 1 gradient_accumulation_steps: 4 micro_batch_size: 4 eval_batch_size: 4 warmup_steps: 0 optimizer: came_pytorch optim_args: enable_stochastic_rounding: true enable_cautious: true enable_8bit: true lr_scheduler: rex learning_rate: 2.5e-7 cosine_min_lr_ratio: 0.05 weight_decay: 0.01 max_grad_norm: 0.5 logging_steps: 1 # Model optimization gradient_checkpointing: offload sdp_attention: true plugins: - axolotl.integrations.liger.LigerPlugin - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin cut_cross_entropy: true liger_rope: true liger_rms_norm: true liger_layer_norm: true liger_glu_activation: true liger_cross_entropy: false liger_fused_linear_cross_entropy: false lora_mlp_kernel: false lora_qkv_kernel: false lora_o_kernel: false # DeepSpeed deepspeed: # Garbage Collection gc_steps: # Debug config debug: true seed: 42 # Token config special_tokens: bos_token: "" eos_token: "" pad_token: "" tokens: ``` ## Citations
Show Citations ```bib @misc{wolf2020huggingfacestransformersstateoftheartnatural, title={HuggingFace's Transformers: State-of-the-art Natural Language Processing}, author={Thomas Wolf and Lysandre Debut and Victor Sanh and Julien Chaumond and Clement Delangue and Anthony Moi and Pierric Cistac and Tim Rault and Rémi Louf and Morgan Funtowicz and Joe Davison and Sam Shleifer and Patrick von Platen and Clara Ma and Yacine Jernite and Julien Plu and Canwen Xu and Teven Le Scao and Sylvain Gugger and Mariama Drame and Quentin Lhoest and Alexander M. Rush}, year={2020}, eprint={1910.03771}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/1910.03771}, } @misc{gemmateam2025gemma3technicalreport, title={Gemma 3 Technical Report}, author={Gemma Team and Aishwarya Kamath and Johan Ferret and Shreya Pathak and Nino Vieillard and Ramona Merhej and Sarah Perrin and Tatiana Matejovicova and Alexandre Ramé and Morgane Rivière and Louis Rouillard and Thomas Mesnard and Geoffrey Cideron and Jean-bastien Grill and Sabela Ramos and Edouard Yvinec and Michelle Casbon and Etienne Pot and Ivo Penchev and Gaël Liu and Francesco Visin and Kathleen Kenealy and Lucas Beyer and Xiaohai Zhai and Anton Tsitsulin and Robert Busa-Fekete and Alex Feng and Noveen Sachdeva and Benjamin Coleman and Yi Gao and Basil Mustafa and Iain Barr and Emilio Parisotto and David Tian and Matan Eyal and Colin Cherry and Jan-Thorsten Peter and Danila Sinopalnikov and Surya Bhupatiraju and Rishabh Agarwal and Mehran Kazemi and Dan Malkin and Ravin Kumar and David Vilar and Idan Brusilovsky and Jiaming Luo and Andreas Steiner and Abe Friesen and Abhanshu Sharma and Abheesht Sharma and Adi Mayrav Gilady and Adrian Goedeckemeyer and Alaa Saade and Alex Feng and Alexander Kolesnikov and Alexei Bendebury and Alvin Abdagic and Amit Vadi and András György and André Susano Pinto and Anil Das and Ankur Bapna and Antoine Miech and Antoine Yang and Antonia Paterson and Ashish Shenoy and Ayan Chakrabarti and Bilal Piot and Bo Wu and Bobak Shahriari and Bryce Petrini and Charlie Chen and Charline Le Lan and Christopher A. Choquette-Choo and CJ Carey and Cormac Brick and Daniel Deutsch and Danielle Eisenbud and Dee Cattle and Derek Cheng and Dimitris Paparas and Divyashree Shivakumar Sreepathihalli and Doug Reid and Dustin Tran and Dustin Zelle and Eric Noland and Erwin Huizenga and Eugene Kharitonov and Frederick Liu and Gagik Amirkhanyan and Glenn Cameron and Hadi Hashemi and Hanna Klimczak-Plucińska and Harman Singh and Harsh Mehta and Harshal Tushar Lehri and Hussein Hazimeh and Ian Ballantyne and Idan Szpektor and Ivan Nardini and Jean Pouget-Abadie and Jetha Chan and Joe Stanton and John Wieting and Jonathan Lai and Jordi Orbay and Joseph Fernandez and Josh Newlan and Ju-yeong Ji and Jyotinder Singh and Kat Black and Kathy Yu and Kevin Hui and Kiran Vodrahalli and Klaus Greff and Linhai Qiu and Marcella Valentine and Marina Coelho and Marvin Ritter and Matt Hoffman and Matthew Watson and Mayank Chaturvedi and Michael Moynihan and Min Ma and Nabila Babar and Natasha Noy and Nathan Byrd and Nick Roy and Nikola Momchev and Nilay Chauhan and Noveen Sachdeva and Oskar Bunyan and Pankil Botarda and Paul Caron and Paul Kishan Rubenstein and Phil Culliton and Philipp Schmid and Pier Giuseppe Sessa and Pingmei Xu and Piotr Stanczyk and Pouya Tafti and Rakesh Shivanna and Renjie Wu and Renke Pan and Reza Rokni and Rob Willoughby and Rohith Vallu and Ryan Mullins and Sammy Jerome and Sara Smoot and Sertan Girgin and Shariq Iqbal and Shashir Reddy and Shruti Sheth and Siim Põder and Sijal Bhatnagar and Sindhu Raghuram Panyam and Sivan Eiger and Susan Zhang and Tianqi Liu and Trevor Yacovone and Tyler Liechty and Uday Kalra and Utku Evci and Vedant Misra and Vincent Roseberry and Vlad Feinberg and Vlad Kolesnikov and Woohyun Han and Woosuk Kwon and Xi Chen and Yinlam Chow and Yuvein Zhu and Zichuan Wei and Zoltan Egyed and Victor Cotruta and Minh Giang and Phoebe Kirk and Anand Rao and Kat Black and Nabila Babar and Jessica Lo and Erica Moreira and Luiz Gustavo Martins and Omar Sanseviero and Lucas Gonzalez and Zach Gleicher and Tris Warkentin and Vahab Mirrokni and Evan Senter and Eli Collins and Joelle Barral and Zoubin Ghahramani and Raia Hadsell and Yossi Matias and D. Sculley and Slav Petrov and Noah Fiedel and Noam Shazeer and Oriol Vinyals and Jeff Dean and Demis Hassabis and Koray Kavukcuoglu and Clement Farabet and Elena Buchatskaya and Jean-Baptiste Alayrac and Rohan Anil and Dmitry and Lepikhin and Sebastian Borgeaud and Olivier Bachem and Armand Joulin and Alek Andreev and Cassidy Hardin and Robert Dadashi and Léonard Hussenot}, year={2025}, eprint={2503.19786}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2503.19786}, } @misc{hu2021loralowrankadaptationlarge, title={LoRA: Low-Rank Adaptation of Large Language Models}, author={Edward J. Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen}, year={2021}, eprint={2106.09685}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2106.09685}, } @misc{dettmers2023qloraefficientfinetuningquantized, title={QLoRA: Efficient Finetuning of Quantized LLMs}, author={Tim Dettmers and Artidoro Pagnoni and Ari Holtzman and Luke Zettlemoyer}, year={2023}, eprint={2305.14314}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2305.14314}, } @misc{dao2023flashattention2fasterattentionbetter, title={FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning}, author={Tri Dao}, year={2023}, eprint={2307.08691}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2307.08691}, } @misc{hsu2024ligerkernelefficienttriton, title={Liger Kernel: Efficient Triton Kernels for LLM Training}, author={Pin-Lun Hsu and Yun Dai and Vignesh Kothapalli and Qingquan Song and Shao Tang and Siyu Zhu and Steven Shimizu and Shivam Sahni and Haowen Ning and Yanning Chen}, year={2024}, eprint={2410.10989}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2410.10989}, } @misc{wijmans2025cutlosseslargevocabularylanguage, title={Cut Your Losses in Large-Vocabulary Language Models}, author={Erik Wijmans and Brody Huval and Alexander Hertzberg and Vladlen Koltun and Philipp Krähenbühl}, year={2025}, eprint={2411.09009}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2411.09009}, } @misc{chen2021rexrevisitingbudgetedtraining, title={REX: Revisiting Budgeted Training with an Improved Schedule}, author={John Chen and Cameron Wolfe and Anastasios Kyrillidis}, year={2021}, eprint={2107.04197}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2107.04197}, } @misc{luo2023cameconfidenceguidedadaptivememory, title={CAME: Confidence-guided Adaptive Memory Efficient Optimization}, author={Yang Luo and Xiaozhe Ren and Zangwei Zheng and Zhuo Jiang and Xin Jiang and Yang You}, year={2023}, eprint={2307.02047}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2307.02047}, } @misc{zamirai2021revisitingbfloat16training, title={Revisiting BFloat16 Training}, author={Pedram Zamirai and Jian Zhang and Christopher R. Aberger and Christopher De Sa}, year={2021}, eprint={2010.06192}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2010.06192}, } @misc{liang2025cautiousoptimizersimprovingtraining, title={Cautious Optimizers: Improving Training with One Line of Code}, author={Kaizhao Liang and Lizhang Chen and Bo Liu and Qiang Liu}, year={2025}, eprint={2411.16085}, archivePrefix={arXiv}, primaryClass={cs.LG}, url={https://arxiv.org/abs/2411.16085}, } @misc{xie2025sana15efficientscaling, title={SANA 1.5: Efficient Scaling of Training-Time and Inference-Time Compute in Linear Diffusion Transformer}, author={Enze Xie and Junsong Chen and Yuyang Zhao and Jincheng Yu and Ligeng Zhu and Chengyue Wu and Yujun Lin and Zhekai Zhang and Muyang Li and Junyu Chen and Han Cai and Bingchen Liu and Daquan Zhou and Song Han}, year={2025}, eprint={2501.18427}, archivePrefix={arXiv}, primaryClass={cs.CV}, url={https://arxiv.org/abs/2501.18427}, } @misc{dallabetta2024fundussimpletousenewsscraper, title={Fundus: A Simple-to-Use News Scraper Optimized for High Quality Extractions}, author={Max Dallabetta and Conrad Dobberstein and Adrian Breiding and Alan Akbik}, year={2024}, eprint={2403.15279}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2403.15279}, } @misc{gosling2023pippapartiallysyntheticconversational, title={PIPPA: A Partially Synthetic Conversational Dataset}, author={Tear Gosling and Alpin Dale and Yinhe Zheng}, year={2023}, eprint={2308.05884}, archivePrefix={arXiv}, primaryClass={cs.CL}, url={https://arxiv.org/abs/2308.05884}, } ```