Upload 15 files

README.md

@@ -0,0 +1,58 @@
+---
+library_name: peft
+license: other
+base_model: deepseek-ai/deepseek-llm-7b-base
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+model-index:
+- name: train_2025-04-22-23-24-06
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_2025-04-22-23-24-06
+
+This model is a fine-tuned version of [deepseek-ai/deepseek-llm-7b-base](https://huggingface.co/deepseek-ai/deepseek-llm-7b-base) on the alpaca_en_demo dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 16
+- optimizer: Use OptimizerNames.ADAMW_TORCH with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.14.0
+- Transformers 4.51.3
+- Pytorch 2.6.0+cu124
+- Datasets 3.5.0
+- Tokenizers 0.21.1

adapter_config.json

@@ -0,0 +1,37 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "deepseek-ai/deepseek-llm-7b-base",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "q_proj",
+    "gate_proj",
+    "k_proj",
+    "down_proj",
+    "v_proj",
+    "up_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a9ab21915f63a44a747df3654bba639aaeb3a757650a8b7193377a775ebe414
+size 75012288

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9908256880733948,
+    "num_input_tokens_seen": 194864,
+    "total_flos": 7611007265931264.0,
+    "train_loss": 0.34003226710445106,
+    "train_runtime": 5382.7216,
+    "train_samples_per_second": 0.728,
+    "train_steps_per_second": 0.045
+}

llamaboard_config.yaml

@@ -0,0 +1,78 @@
+top.booster: auto
+top.checkpoint_path: []
+top.finetuning_type: lora
+top.model_name: DeepSeek-LLM-7B-Base
+top.quantization_bit: '4'
+top.quantization_method: bnb
+top.rope_scaling: none
+top.template: default
+train.additional_target: ''
+train.apollo_rank: 16
+train.apollo_scale: 32
+train.apollo_target: all
+train.apollo_update_interval: 200
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 2
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 256
+train.dataset:
+- alpaca_en_demo
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: none
+train.extra_args: '{"optim": "adamw_torch"}'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 2
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 2e-4
+train.logging_steps: 5
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.mask_history: false
+train.max_grad_norm: '0.3'
+train.max_samples: '10000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '3.0'
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to:
+- none
+train.resize_vocab: false
+train.reward_model: []
+train.save_steps: 100
+train.swanlab_api_key: ''
+train.swanlab_link: ''
+train.swanlab_mode: cloud
+train.swanlab_project: llamafactory
+train.swanlab_run_name: ''
+train.swanlab_workspace: ''
+train.train_on_prompt: false
+train.training_stage: Supervised Fine-Tuning
+train.use_apollo: false
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.use_swanlab: false
+train.val_size: 0
+train.warmup_steps: 0

running_log.txt

@@ -0,0 +1,424 @@
+[INFO|2025-04-22 23:31:16] tokenization_utils_base.py:2060 >> loading file tokenizer.model from cache at None
+
+[INFO|2025-04-22 23:31:16] tokenization_utils_base.py:2060 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/tokenizer.json
+
+[INFO|2025-04-22 23:31:16] tokenization_utils_base.py:2060 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-04-22 23:31:16] tokenization_utils_base.py:2060 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-04-22 23:31:16] tokenization_utils_base.py:2060 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/tokenizer_config.json
+
+[INFO|2025-04-22 23:31:16] tokenization_utils_base.py:2060 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-04-22 23:31:17] tokenization_utils_base.py:2323 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-04-22 23:31:17] configuration_utils.py:693 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/config.json
+
+[INFO|2025-04-22 23:31:17] configuration_utils.py:765 >> Model config LlamaConfig {
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-04-22 23:31:17] tokenization_utils_base.py:2060 >> loading file tokenizer.model from cache at None
+
+[INFO|2025-04-22 23:31:17] tokenization_utils_base.py:2060 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/tokenizer.json
+
+[INFO|2025-04-22 23:31:17] tokenization_utils_base.py:2060 >> loading file added_tokens.json from cache at None
+
+[INFO|2025-04-22 23:31:17] tokenization_utils_base.py:2060 >> loading file special_tokens_map.json from cache at None
+
+[INFO|2025-04-22 23:31:17] tokenization_utils_base.py:2060 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/tokenizer_config.json
+
+[INFO|2025-04-22 23:31:17] tokenization_utils_base.py:2060 >> loading file chat_template.jinja from cache at None
+
+[INFO|2025-04-22 23:31:17] tokenization_utils_base.py:2323 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|2025-04-22 23:31:18] logging.py:143 >> Loading dataset alpaca_en_demo.json...
+
+[INFO|2025-04-22 23:31:28] configuration_utils.py:693 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/config.json
+
+[INFO|2025-04-22 23:31:28] configuration_utils.py:765 >> Model config LlamaConfig {
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-04-22 23:31:28] logging.py:143 >> Quantizing model to 4 bit with bitsandbytes.
+
+[INFO|2025-04-22 23:31:28] logging.py:143 >> KV cache is disabled during training.
+
+[INFO|2025-04-22 23:31:29] modeling_utils.py:1124 >> loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/pytorch_model.bin.index.json
+
+[INFO|2025-04-22 23:31:30] safetensors_conversion.py:61 >> Attempting to create safetensors variant
+
+[INFO|2025-04-22 23:31:30] safetensors_conversion.py:74 >> Safetensors PR exists
+
+[INFO|2025-04-22 23:33:11] modeling_utils.py:2167 >> Instantiating LlamaForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|2025-04-22 23:33:11] configuration_utils.py:1142 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "use_cache": false
+}
+
+
+[INFO|2025-04-22 23:34:25] modeling_utils.py:4930 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
+
+
+[INFO|2025-04-22 23:34:25] modeling_utils.py:4938 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at deepseek-ai/deepseek-llm-7b-base.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
+
+[INFO|2025-04-22 23:34:25] configuration_utils.py:1097 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/generation_config.json
+
+[INFO|2025-04-22 23:34:25] configuration_utils.py:1142 >> Generate config GenerationConfig {
+  "bos_token_id": 100000,
+  "eos_token_id": 100001
+}
+
+
+[INFO|2025-04-22 23:34:25] logging.py:143 >> Gradient checkpointing enabled.
+
+[INFO|2025-04-22 23:34:25] logging.py:143 >> Using torch SDPA for faster training and inference.
+
+[INFO|2025-04-22 23:34:25] logging.py:143 >> Upcasting trainable params to float32.
+
+[INFO|2025-04-22 23:34:25] logging.py:143 >> Fine-tuning method: LoRA
+
+[INFO|2025-04-22 23:34:25] logging.py:143 >> Found linear modules: o_proj,q_proj,gate_proj,k_proj,down_proj,v_proj,up_proj
+
+[INFO|2025-04-22 23:34:26] logging.py:143 >> trainable params: 18,739,200 || all params: 6,929,104,896 || trainable%: 0.2704
+
+[INFO|2025-04-22 23:34:26] trainer.py:748 >> Using auto half precision backend
+
+[INFO|2025-04-22 23:34:26] trainer.py:2414 >> ***** Running training *****
+
+[INFO|2025-04-22 23:34:26] trainer.py:2415 >>   Num examples = 1,307
+
+[INFO|2025-04-22 23:34:26] trainer.py:2416 >>   Num Epochs = 3
+
+[INFO|2025-04-22 23:34:26] trainer.py:2417 >>   Instantaneous batch size per device = 2
+
+[INFO|2025-04-22 23:34:26] trainer.py:2420 >>   Total train batch size (w. parallel, distributed & accumulation) = 16
+
+[INFO|2025-04-22 23:34:26] trainer.py:2421 >>   Gradient Accumulation steps = 8
+
+[INFO|2025-04-22 23:34:26] trainer.py:2422 >>   Total optimization steps = 243
+
+[INFO|2025-04-22 23:34:26] trainer.py:2423 >>   Number of trainable parameters = 18,739,200
+
+[INFO|2025-04-22 23:36:16] logging.py:143 >> {'loss': 2.3883, 'learning_rate': 1.9987e-04, 'epoch': 0.06, 'throughput': 36.53}
+
+[INFO|2025-04-22 23:38:05] logging.py:143 >> {'loss': 1.4752, 'learning_rate': 1.9932e-04, 'epoch': 0.12, 'throughput': 36.29}
+
+[INFO|2025-04-22 23:39:57] logging.py:143 >> {'loss': 1.1174, 'learning_rate': 1.9837e-04, 'epoch': 0.18, 'throughput': 36.12}
+
+[INFO|2025-04-22 23:41:45] logging.py:143 >> {'loss': 0.7815, 'learning_rate': 1.9700e-04, 'epoch': 0.24, 'throughput': 36.03}
+
+[INFO|2025-04-22 23:43:35] logging.py:143 >> {'loss': 0.6320, 'learning_rate': 1.9522e-04, 'epoch': 0.31, 'throughput': 36.22}
+
+[INFO|2025-04-22 23:45:25] logging.py:143 >> {'loss': 0.4603, 'learning_rate': 1.9305e-04, 'epoch': 0.37, 'throughput': 36.20}
+
+[INFO|2025-04-22 23:47:17] logging.py:143 >> {'loss': 0.4761, 'learning_rate': 1.9049e-04, 'epoch': 0.43, 'throughput': 36.25}
+
+[INFO|2025-04-22 23:49:06] logging.py:143 >> {'loss': 0.4525, 'learning_rate': 1.8756e-04, 'epoch': 0.49, 'throughput': 36.22}
+
+[INFO|2025-04-22 23:50:55] logging.py:143 >> {'loss': 0.4803, 'learning_rate': 1.8425e-04, 'epoch': 0.55, 'throughput': 36.12}
+
+[INFO|2025-04-22 23:52:45] logging.py:143 >> {'loss': 0.3933, 'learning_rate': 1.8060e-04, 'epoch': 0.61, 'throughput': 36.11}
+
+[INFO|2025-04-22 23:54:39] logging.py:143 >> {'loss': 0.3919, 'learning_rate': 1.7660e-04, 'epoch': 0.67, 'throughput': 36.22}
+
+[INFO|2025-04-22 23:56:33] logging.py:143 >> {'loss': 0.3804, 'learning_rate': 1.7229e-04, 'epoch': 0.73, 'throughput': 36.31}
+
+[INFO|2025-04-22 23:58:22] logging.py:143 >> {'loss': 0.3576, 'learning_rate': 1.6768e-04, 'epoch': 0.80, 'throughput': 36.28}
+
+[INFO|2025-04-23 00:00:11] logging.py:143 >> {'loss': 0.3343, 'learning_rate': 1.6278e-04, 'epoch': 0.86, 'throughput': 36.32}
+
+[INFO|2025-04-23 00:01:59] logging.py:143 >> {'loss': 0.4526, 'learning_rate': 1.5762e-04, 'epoch': 0.92, 'throughput': 36.27}
+
+[INFO|2025-04-23 00:03:48] logging.py:143 >> {'loss': 0.4048, 'learning_rate': 1.5222e-04, 'epoch': 0.98, 'throughput': 36.24}
+
+[INFO|2025-04-23 00:05:46] logging.py:143 >> {'loss': 0.3599, 'learning_rate': 1.4660e-04, 'epoch': 1.05, 'throughput': 36.26}
+
+[INFO|2025-04-23 00:07:38] logging.py:143 >> {'loss': 0.2197, 'learning_rate': 1.4079e-04, 'epoch': 1.11, 'throughput': 36.31}
+
+[INFO|2025-04-23 00:09:28] logging.py:143 >> {'loss': 0.1816, 'learning_rate': 1.3481e-04, 'epoch': 1.17, 'throughput': 36.33}
+
+[INFO|2025-04-23 00:11:19] logging.py:143 >> {'loss': 0.1839, 'learning_rate': 1.2868e-04, 'epoch': 1.23, 'throughput': 36.33}
+
+[INFO|2025-04-23 00:11:19] trainer.py:3984 >> Saving model checkpoint to saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-100
+
+[INFO|2025-04-23 00:11:19] configuration_utils.py:693 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/config.json
+
+[INFO|2025-04-23 00:11:19] configuration_utils.py:765 >> Model config LlamaConfig {
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-04-23 00:11:19] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-100/tokenizer_config.json
+
+[INFO|2025-04-23 00:11:19] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-100/special_tokens_map.json
+
+[INFO|2025-04-23 00:13:14] logging.py:143 >> {'loss': 0.2100, 'learning_rate': 1.2243e-04, 'epoch': 1.29, 'throughput': 36.25}
+
+[INFO|2025-04-23 00:15:04] logging.py:143 >> {'loss': 0.2568, 'learning_rate': 1.1609e-04, 'epoch': 1.35, 'throughput': 36.24}
+
+[INFO|2025-04-23 00:16:56] logging.py:143 >> {'loss': 0.2049, 'learning_rate': 1.0968e-04, 'epoch': 1.42, 'throughput': 36.27}
+
+[INFO|2025-04-23 00:18:44] logging.py:143 >> {'loss': 0.2213, 'learning_rate': 1.0323e-04, 'epoch': 1.48, 'throughput': 36.28}
+
+[INFO|2025-04-23 00:20:34] logging.py:143 >> {'loss': 0.2836, 'learning_rate': 9.6768e-05, 'epoch': 1.54, 'throughput': 36.28}
+
+[INFO|2025-04-23 00:22:25] logging.py:143 >> {'loss': 0.2280, 'learning_rate': 9.0319e-05, 'epoch': 1.60, 'throughput': 36.28}
+
+[INFO|2025-04-23 00:24:15] logging.py:143 >> {'loss': 0.1778, 'learning_rate': 8.3910e-05, 'epoch': 1.66, 'throughput': 36.28}
+
+[INFO|2025-04-23 00:26:03] logging.py:143 >> {'loss': 0.1871, 'learning_rate': 7.7568e-05, 'epoch': 1.72, 'throughput': 36.27}
+
+[INFO|2025-04-23 00:27:52] logging.py:143 >> {'loss': 0.2289, 'learning_rate': 7.1320e-05, 'epoch': 1.78, 'throughput': 36.27}
+
+[INFO|2025-04-23 00:29:41] logging.py:143 >> {'loss': 0.2889, 'learning_rate': 6.5191e-05, 'epoch': 1.84, 'throughput': 36.27}
+
+[INFO|2025-04-23 00:31:32] logging.py:143 >> {'loss': 0.2724, 'learning_rate': 5.9208e-05, 'epoch': 1.91, 'throughput': 36.25}
+
+[INFO|2025-04-23 00:33:23] logging.py:143 >> {'loss': 0.2359, 'learning_rate': 5.3396e-05, 'epoch': 1.97, 'throughput': 36.26}
+
+[INFO|2025-04-23 00:35:23] logging.py:143 >> {'loss': 0.1864, 'learning_rate': 4.7778e-05, 'epoch': 2.04, 'throughput': 36.26}
+
+[INFO|2025-04-23 00:37:13] logging.py:143 >> {'loss': 0.0671, 'learning_rate': 4.2378e-05, 'epoch': 2.10, 'throughput': 36.26}
+
+[INFO|2025-04-23 00:39:02] logging.py:143 >> {'loss': 0.1465, 'learning_rate': 3.7219e-05, 'epoch': 2.16, 'throughput': 36.23}
+
+[INFO|2025-04-23 00:40:51] logging.py:143 >> {'loss': 0.0910, 'learning_rate': 3.2322e-05, 'epoch': 2.22, 'throughput': 36.22}
+
+[INFO|2025-04-23 00:42:41] logging.py:143 >> {'loss': 0.1140, 'learning_rate': 2.7708e-05, 'epoch': 2.28, 'throughput': 36.21}
+
+[INFO|2025-04-23 00:44:33] logging.py:143 >> {'loss': 0.1381, 'learning_rate': 2.3396e-05, 'epoch': 2.34, 'throughput': 36.23}
+
+[INFO|2025-04-23 00:46:22] logging.py:143 >> {'loss': 0.0799, 'learning_rate': 1.9403e-05, 'epoch': 2.40, 'throughput': 36.24}
+
+[INFO|2025-04-23 00:48:14] logging.py:143 >> {'loss': 0.0722, 'learning_rate': 1.5748e-05, 'epoch': 2.46, 'throughput': 36.25}
+
+[INFO|2025-04-23 00:48:14] trainer.py:3984 >> Saving model checkpoint to saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-200
+
+[INFO|2025-04-23 00:48:14] configuration_utils.py:693 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/config.json
+
+[INFO|2025-04-23 00:48:14] configuration_utils.py:765 >> Model config LlamaConfig {
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-04-23 00:48:15] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-200/tokenizer_config.json
+
+[INFO|2025-04-23 00:48:15] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-200/special_tokens_map.json
+
+[INFO|2025-04-23 00:50:13] logging.py:143 >> {'loss': 0.1003, 'learning_rate': 1.2444e-05, 'epoch': 2.53, 'throughput': 36.23}
+
+[INFO|2025-04-23 00:52:01] logging.py:143 >> {'loss': 0.1155, 'learning_rate': 9.5063e-06, 'epoch': 2.59, 'throughput': 36.21}
+
+[INFO|2025-04-23 00:53:51] logging.py:143 >> {'loss': 0.0962, 'learning_rate': 6.9464e-06, 'epoch': 2.65, 'throughput': 36.22}
+
+[INFO|2025-04-23 00:55:43] logging.py:143 >> {'loss': 0.0808, 'learning_rate': 4.7752e-06, 'epoch': 2.71, 'throughput': 36.23}
+
+[INFO|2025-04-23 00:57:33] logging.py:143 >> {'loss': 0.1206, 'learning_rate': 3.0018e-06, 'epoch': 2.77, 'throughput': 36.22}
+
+[INFO|2025-04-23 00:59:23] logging.py:143 >> {'loss': 0.1259, 'learning_rate': 1.6335e-06, 'epoch': 2.83, 'throughput': 36.22}
+
+[INFO|2025-04-23 01:01:11] logging.py:143 >> {'loss': 0.1580, 'learning_rate': 6.7616e-07, 'epoch': 2.89, 'throughput': 36.22}
+
+[INFO|2025-04-23 01:02:59] logging.py:143 >> {'loss': 0.0620, 'learning_rate': 1.3368e-07, 'epoch': 2.95, 'throughput': 36.21}
+
+[INFO|2025-04-23 01:04:07] trainer.py:3984 >> Saving model checkpoint to saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-243
+
+[INFO|2025-04-23 01:04:07] configuration_utils.py:693 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/config.json
+
+[INFO|2025-04-23 01:04:07] configuration_utils.py:765 >> Model config LlamaConfig {
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-04-23 01:04:08] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-243/tokenizer_config.json
+
+[INFO|2025-04-23 01:04:08] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/checkpoint-243/special_tokens_map.json
+
+[INFO|2025-04-23 01:04:09] trainer.py:2681 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|2025-04-23 01:04:09] trainer.py:3984 >> Saving model checkpoint to saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06
+
+[INFO|2025-04-23 01:04:09] configuration_utils.py:693 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--deepseek-ai--deepseek-llm-7b-base/snapshots/7683fea62db869066ddaff6a41d032262c490d4f/config.json
+
+[INFO|2025-04-23 01:04:09] configuration_utils.py:765 >> Model config LlamaConfig {
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 30,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "vocab_size": 102400
+}
+
+
+[INFO|2025-04-23 01:04:09] tokenization_utils_base.py:2510 >> tokenizer config file saved in saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/tokenizer_config.json
+
+[INFO|2025-04-23 01:04:09] tokenization_utils_base.py:2519 >> Special tokens file saved in saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06/special_tokens_map.json
+
+[WARNING|2025-04-23 01:04:10] logging.py:148 >> No metric eval_loss to plot.
+
+[WARNING|2025-04-23 01:04:10] logging.py:148 >> No metric eval_accuracy to plot.
+
+[INFO|2025-04-23 01:04:10] modelcard.py:450 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,141 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "100000": {
+      "content": "<｜begin▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100001": {
+      "content": "<｜end▁of▁sentence｜>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100002": {
+      "content": "ø",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100003": {
+      "content": "ö",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100004": {
+      "content": "ú",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100005": {
+      "content": "ÿ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100006": {
+      "content": "õ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100007": {
+      "content": "÷",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100008": {
+      "content": "û",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100009": {
+      "content": "ý",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100010": {
+      "content": "À",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100011": {
+      "content": "ù",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100012": {
+      "content": "Á",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100013": {
+      "content": "þ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100014": {
+      "content": "ü",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ 'System: ' + system_message + '<｜end▁of▁sentence｜>' + '\n' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '<｜end▁of▁sentence｜>' + '\nAssistant:' }}{% elif message['role'] == 'assistant' %}{{ content + '<｜end▁of▁sentence｜>' + '\n' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<｜end▁of▁sentence｜>",
+  "extra_special_tokens": {},
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizerFast",
+  "unk_token": null,
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 2.9908256880733948,
+    "num_input_tokens_seen": 194864,
+    "total_flos": 7611007265931264.0,
+    "train_loss": 0.34003226710445106,
+    "train_runtime": 5382.7216,
+    "train_samples_per_second": 0.728,
+    "train_steps_per_second": 0.045
+}

trainer_log.jsonl

@@ -0,0 +1,49 @@
+{"current_steps": 5, "total_steps": 243, "loss": 2.3883, "lr": 0.00019986631570270832, "epoch": 0.06116207951070336, "percentage": 2.06, "elapsed_time": "0:01:49", "remaining_time": "1:27:12", "throughput": 36.53, "total_tokens": 4016}
+{"current_steps": 10, "total_steps": 243, "loss": 1.4752, "lr": 0.00019932383577419432, "epoch": 0.12232415902140673, "percentage": 4.12, "elapsed_time": "0:03:39", "remaining_time": "1:25:04", "throughput": 36.29, "total_tokens": 7952}
+{"current_steps": 15, "total_steps": 243, "loss": 1.1174, "lr": 0.0001983664691986601, "epoch": 0.1834862385321101, "percentage": 6.17, "elapsed_time": "0:05:30", "remaining_time": "1:23:42", "throughput": 36.12, "total_tokens": 11936}
+{"current_steps": 20, "total_steps": 243, "loss": 0.7815, "lr": 0.00019699821500217434, "epoch": 0.24464831804281345, "percentage": 8.23, "elapsed_time": "0:07:18", "remaining_time": "1:21:32", "throughput": 36.03, "total_tokens": 15808}
+{"current_steps": 25, "total_steps": 243, "loss": 0.632, "lr": 0.00019522478853384155, "epoch": 0.3058103975535168, "percentage": 10.29, "elapsed_time": "0:09:09", "remaining_time": "1:19:48", "throughput": 36.22, "total_tokens": 19888}
+{"current_steps": 30, "total_steps": 243, "loss": 0.4603, "lr": 0.00019305359759215685, "epoch": 0.3669724770642202, "percentage": 12.35, "elapsed_time": "0:10:59", "remaining_time": "1:17:59", "throughput": 36.2, "total_tokens": 23856}
+{"current_steps": 35, "total_steps": 243, "loss": 0.4761, "lr": 0.00019049371148181253, "epoch": 0.42813455657492355, "percentage": 14.4, "elapsed_time": "0:12:50", "remaining_time": "1:16:19", "throughput": 36.25, "total_tokens": 27936}
+{"current_steps": 40, "total_steps": 243, "loss": 0.4525, "lr": 0.0001875558231302091, "epoch": 0.4892966360856269, "percentage": 16.46, "elapsed_time": "0:14:39", "remaining_time": "1:14:25", "throughput": 36.22, "total_tokens": 31872}
+{"current_steps": 45, "total_steps": 243, "loss": 0.4803, "lr": 0.00018425220442191495, "epoch": 0.5504587155963303, "percentage": 18.52, "elapsed_time": "0:16:28", "remaining_time": "1:12:28", "throughput": 36.12, "total_tokens": 35696}
+{"current_steps": 50, "total_steps": 243, "loss": 0.3933, "lr": 0.00018059665493764743, "epoch": 0.6116207951070336, "percentage": 20.58, "elapsed_time": "0:18:18", "remaining_time": "1:10:41", "throughput": 36.11, "total_tokens": 39680}
+{"current_steps": 55, "total_steps": 243, "loss": 0.3919, "lr": 0.0001766044443118978, "epoch": 0.672782874617737, "percentage": 22.63, "elapsed_time": "0:20:12", "remaining_time": "1:09:05", "throughput": 36.22, "total_tokens": 43920}
+{"current_steps": 60, "total_steps": 243, "loss": 0.3804, "lr": 0.00017229224844997928, "epoch": 0.7339449541284404, "percentage": 24.69, "elapsed_time": "0:22:06", "remaining_time": "1:07:25", "throughput": 36.31, "total_tokens": 48160}
+{"current_steps": 65, "total_steps": 243, "loss": 0.3576, "lr": 0.00016767807987092621, "epoch": 0.7951070336391437, "percentage": 26.75, "elapsed_time": "0:23:55", "remaining_time": "1:05:30", "throughput": 36.28, "total_tokens": 52080}
+{"current_steps": 70, "total_steps": 243, "loss": 0.3343, "lr": 0.00016278121246720987, "epoch": 0.8562691131498471, "percentage": 28.81, "elapsed_time": "0:25:45", "remaining_time": "1:03:39", "throughput": 36.32, "total_tokens": 56128}
+{"current_steps": 75, "total_steps": 243, "loss": 0.4526, "lr": 0.00015762210099555803, "epoch": 0.9174311926605505, "percentage": 30.86, "elapsed_time": "0:27:32", "remaining_time": "1:01:42", "throughput": 36.27, "total_tokens": 59952}
+{"current_steps": 80, "total_steps": 243, "loss": 0.4048, "lr": 0.00015222229563517385, "epoch": 0.9785932721712538, "percentage": 32.92, "elapsed_time": "0:29:21", "remaining_time": "0:59:48", "throughput": 36.24, "total_tokens": 63840}
+{"current_steps": 85, "total_steps": 243, "loss": 0.3599, "lr": 0.0001466043519702539, "epoch": 1.0489296636085628, "percentage": 34.98, "elapsed_time": "0:31:20", "remaining_time": "0:58:14", "throughput": 36.26, "total_tokens": 68176}
+{"current_steps": 90, "total_steps": 243, "loss": 0.2197, "lr": 0.00014079173677281837, "epoch": 1.110091743119266, "percentage": 37.04, "elapsed_time": "0:33:12", "remaining_time": "0:56:26", "throughput": 36.31, "total_tokens": 72336}
+{"current_steps": 95, "total_steps": 243, "loss": 0.1816, "lr": 0.00013480872997940905, "epoch": 1.1712538226299694, "percentage": 39.09, "elapsed_time": "0:35:02", "remaining_time": "0:54:35", "throughput": 36.33, "total_tokens": 76384}
+{"current_steps": 100, "total_steps": 243, "loss": 0.1839, "lr": 0.00012868032327110904, "epoch": 1.2324159021406729, "percentage": 41.15, "elapsed_time": "0:36:52", "remaining_time": "0:52:43", "throughput": 36.33, "total_tokens": 80368}
+{"current_steps": 105, "total_steps": 243, "loss": 0.21, "lr": 0.00012243211568052677, "epoch": 1.2935779816513762, "percentage": 43.21, "elapsed_time": "0:38:48", "remaining_time": "0:50:59", "throughput": 36.25, "total_tokens": 84400}
+{"current_steps": 110, "total_steps": 243, "loss": 0.2568, "lr": 0.00011609020666180575, "epoch": 1.3547400611620795, "percentage": 45.27, "elapsed_time": "0:40:37", "remaining_time": "0:49:07", "throughput": 36.24, "total_tokens": 88352}
+{"current_steps": 115, "total_steps": 243, "loss": 0.2049, "lr": 0.00010968108707031792, "epoch": 1.4159021406727827, "percentage": 47.33, "elapsed_time": "0:42:29", "remaining_time": "0:47:17", "throughput": 36.27, "total_tokens": 92464}
+{"current_steps": 120, "total_steps": 243, "loss": 0.2213, "lr": 0.00010323152850743107, "epoch": 1.4770642201834863, "percentage": 49.38, "elapsed_time": "0:44:18", "remaining_time": "0:45:24", "throughput": 36.28, "total_tokens": 96432}
+{"current_steps": 125, "total_steps": 243, "loss": 0.2836, "lr": 9.676847149256895e-05, "epoch": 1.5382262996941896, "percentage": 51.44, "elapsed_time": "0:46:08", "remaining_time": "0:43:33", "throughput": 36.28, "total_tokens": 100416}
+{"current_steps": 130, "total_steps": 243, "loss": 0.228, "lr": 9.03189129296821e-05, "epoch": 1.599388379204893, "percentage": 53.5, "elapsed_time": "0:47:58", "remaining_time": "0:41:42", "throughput": 36.28, "total_tokens": 104432}
+{"current_steps": 135, "total_steps": 243, "loss": 0.1778, "lr": 8.390979333819426e-05, "epoch": 1.6605504587155964, "percentage": 55.56, "elapsed_time": "0:49:48", "remaining_time": "0:39:50", "throughput": 36.28, "total_tokens": 108432}
+{"current_steps": 140, "total_steps": 243, "loss": 0.1871, "lr": 7.756788431947326e-05, "epoch": 1.7217125382262997, "percentage": 57.61, "elapsed_time": "0:51:36", "remaining_time": "0:37:58", "throughput": 36.27, "total_tokens": 112320}
+{"current_steps": 145, "total_steps": 243, "loss": 0.2289, "lr": 7.131967672889101e-05, "epoch": 1.782874617737003, "percentage": 59.67, "elapsed_time": "0:53:26", "remaining_time": "0:36:06", "throughput": 36.27, "total_tokens": 116272}
+{"current_steps": 150, "total_steps": 243, "loss": 0.2889, "lr": 6.519127002059095e-05, "epoch": 1.8440366972477065, "percentage": 61.73, "elapsed_time": "0:55:14", "remaining_time": "0:34:15", "throughput": 36.27, "total_tokens": 120224}
+{"current_steps": 155, "total_steps": 243, "loss": 0.2724, "lr": 5.920826322718165e-05, "epoch": 1.9051987767584098, "percentage": 63.79, "elapsed_time": "0:57:05", "remaining_time": "0:32:24", "throughput": 36.25, "total_tokens": 124176}
+{"current_steps": 160, "total_steps": 243, "loss": 0.2359, "lr": 5.339564802974615e-05, "epoch": 1.9663608562691133, "percentage": 65.84, "elapsed_time": "0:58:56", "remaining_time": "0:30:34", "throughput": 36.26, "total_tokens": 128256}
+{"current_steps": 165, "total_steps": 243, "loss": 0.1864, "lr": 4.777770436482617e-05, "epoch": 2.036697247706422, "percentage": 67.9, "elapsed_time": "1:00:56", "remaining_time": "0:28:48", "throughput": 36.26, "total_tokens": 132576}
+{"current_steps": 170, "total_steps": 243, "loss": 0.0671, "lr": 4.2377899004441966e-05, "epoch": 2.0978593272171255, "percentage": 69.96, "elapsed_time": "1:02:47", "remaining_time": "0:26:57", "throughput": 36.26, "total_tokens": 136608}
+{"current_steps": 175, "total_steps": 243, "loss": 0.1465, "lr": 3.721878753279017e-05, "epoch": 2.159021406727829, "percentage": 72.02, "elapsed_time": "1:04:35", "remaining_time": "0:25:05", "throughput": 36.23, "total_tokens": 140416}
+{"current_steps": 180, "total_steps": 243, "loss": 0.091, "lr": 3.2321920129073816e-05, "epoch": 2.220183486238532, "percentage": 74.07, "elapsed_time": "1:06:24", "remaining_time": "0:23:14", "throughput": 36.22, "total_tokens": 144336}
+{"current_steps": 185, "total_steps": 243, "loss": 0.114, "lr": 2.770775155002071e-05, "epoch": 2.2813455657492354, "percentage": 76.13, "elapsed_time": "1:08:14", "remaining_time": "0:21:23", "throughput": 36.21, "total_tokens": 148256}
+{"current_steps": 190, "total_steps": 243, "loss": 0.1381, "lr": 2.339555568810221e-05, "epoch": 2.3425076452599387, "percentage": 78.19, "elapsed_time": "1:10:06", "remaining_time": "0:19:33", "throughput": 36.23, "total_tokens": 152416}
+{"current_steps": 195, "total_steps": 243, "loss": 0.0799, "lr": 1.9403345062352573e-05, "epoch": 2.4036697247706424, "percentage": 80.25, "elapsed_time": "1:11:56", "remaining_time": "0:17:42", "throughput": 36.24, "total_tokens": 156400}
+{"current_steps": 200, "total_steps": 243, "loss": 0.0722, "lr": 1.5747795578085046e-05, "epoch": 2.4648318042813457, "percentage": 82.3, "elapsed_time": "1:13:47", "remaining_time": "0:15:52", "throughput": 36.25, "total_tokens": 160528}
+{"current_steps": 205, "total_steps": 243, "loss": 0.1003, "lr": 1.2444176869790925e-05, "epoch": 2.525993883792049, "percentage": 84.36, "elapsed_time": "1:15:46", "remaining_time": "0:14:02", "throughput": 36.23, "total_tokens": 164720}
+{"current_steps": 210, "total_steps": 243, "loss": 0.1155, "lr": 9.506288518187467e-06, "epoch": 2.5871559633027523, "percentage": 86.42, "elapsed_time": "1:17:34", "remaining_time": "0:12:11", "throughput": 36.21, "total_tokens": 168560}
+{"current_steps": 215, "total_steps": 243, "loss": 0.0962, "lr": 6.946402407843155e-06, "epoch": 2.6483180428134556, "percentage": 88.48, "elapsed_time": "1:19:25", "remaining_time": "0:10:20", "throughput": 36.22, "total_tokens": 172608}
+{"current_steps": 220, "total_steps": 243, "loss": 0.0808, "lr": 4.775211466158469e-06, "epoch": 2.709480122324159, "percentage": 90.53, "elapsed_time": "1:21:16", "remaining_time": "0:08:29", "throughput": 36.23, "total_tokens": 176656}
+{"current_steps": 225, "total_steps": 243, "loss": 0.1206, "lr": 3.0017849978256516e-06, "epoch": 2.770642201834862, "percentage": 92.59, "elapsed_time": "1:23:06", "remaining_time": "0:06:38", "throughput": 36.22, "total_tokens": 180608}
+{"current_steps": 230, "total_steps": 243, "loss": 0.1259, "lr": 1.6335308013398886e-06, "epoch": 2.8318042813455655, "percentage": 94.65, "elapsed_time": "1:24:56", "remaining_time": "0:04:48", "throughput": 36.22, "total_tokens": 184592}
+{"current_steps": 235, "total_steps": 243, "loss": 0.158, "lr": 6.761642258056978e-07, "epoch": 2.8929663608562692, "percentage": 96.71, "elapsed_time": "1:26:44", "remaining_time": "0:02:57", "throughput": 36.22, "total_tokens": 188496}
+{"current_steps": 240, "total_steps": 243, "loss": 0.062, "lr": 1.3368429729168076e-07, "epoch": 2.9541284403669725, "percentage": 98.77, "elapsed_time": "1:28:33", "remaining_time": "0:01:06", "throughput": 36.21, "total_tokens": 192368}
+{"current_steps": 243, "total_steps": 243, "epoch": 2.9908256880733948, "percentage": 100.0, "elapsed_time": "1:29:42", "remaining_time": "0:00:00", "throughput": 36.2, "total_tokens": 194864}

trainer_state.json

@@ -0,0 +1,428 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9908256880733948,
+  "eval_steps": 500,
+  "global_step": 243,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.06116207951070336,
+      "grad_norm": 1.7883533239364624,
+      "learning_rate": 0.00019986631570270832,
+      "loss": 2.3883,
+      "num_input_tokens_seen": 4016,
+      "step": 5
+    },
+    {
+      "epoch": 0.12232415902140673,
+      "grad_norm": 2.9251019954681396,
+      "learning_rate": 0.00019932383577419432,
+      "loss": 1.4752,
+      "num_input_tokens_seen": 7952,
+      "step": 10
+    },
+    {
+      "epoch": 0.1834862385321101,
+      "grad_norm": 5.290874004364014,
+      "learning_rate": 0.0001983664691986601,
+      "loss": 1.1174,
+      "num_input_tokens_seen": 11936,
+      "step": 15
+    },
+    {
+      "epoch": 0.24464831804281345,
+      "grad_norm": 6.394321918487549,
+      "learning_rate": 0.00019699821500217434,
+      "loss": 0.7815,
+      "num_input_tokens_seen": 15808,
+      "step": 20
+    },
+    {
+      "epoch": 0.3058103975535168,
+      "grad_norm": 1.3936750888824463,
+      "learning_rate": 0.00019522478853384155,
+      "loss": 0.632,
+      "num_input_tokens_seen": 19888,
+      "step": 25
+    },
+    {
+      "epoch": 0.3669724770642202,
+      "grad_norm": 0.6125549674034119,
+      "learning_rate": 0.00019305359759215685,
+      "loss": 0.4603,
+      "num_input_tokens_seen": 23856,
+      "step": 30
+    },
+    {
+      "epoch": 0.42813455657492355,
+      "grad_norm": 0.8421280980110168,
+      "learning_rate": 0.00019049371148181253,
+      "loss": 0.4761,
+      "num_input_tokens_seen": 27936,
+      "step": 35
+    },
+    {
+      "epoch": 0.4892966360856269,
+      "grad_norm": 0.7261258959770203,
+      "learning_rate": 0.0001875558231302091,
+      "loss": 0.4525,
+      "num_input_tokens_seen": 31872,
+      "step": 40
+    },
+    {
+      "epoch": 0.5504587155963303,
+      "grad_norm": 0.8600367903709412,
+      "learning_rate": 0.00018425220442191495,
+      "loss": 0.4803,
+      "num_input_tokens_seen": 35696,
+      "step": 45
+    },
+    {
+      "epoch": 0.6116207951070336,
+      "grad_norm": 0.8055685758590698,
+      "learning_rate": 0.00018059665493764743,
+      "loss": 0.3933,
+      "num_input_tokens_seen": 39680,
+      "step": 50
+    },
+    {
+      "epoch": 0.672782874617737,
+      "grad_norm": 0.7538366317749023,
+      "learning_rate": 0.0001766044443118978,
+      "loss": 0.3919,
+      "num_input_tokens_seen": 43920,
+      "step": 55
+    },
+    {
+      "epoch": 0.7339449541284404,
+      "grad_norm": 0.8907052874565125,
+      "learning_rate": 0.00017229224844997928,
+      "loss": 0.3804,
+      "num_input_tokens_seen": 48160,
+      "step": 60
+    },
+    {
+      "epoch": 0.7951070336391437,
+      "grad_norm": 0.7426741123199463,
+      "learning_rate": 0.00016767807987092621,
+      "loss": 0.3576,
+      "num_input_tokens_seen": 52080,
+      "step": 65
+    },
+    {
+      "epoch": 0.8562691131498471,
+      "grad_norm": 0.6457409262657166,
+      "learning_rate": 0.00016278121246720987,
+      "loss": 0.3343,
+      "num_input_tokens_seen": 56128,
+      "step": 70
+    },
+    {
+      "epoch": 0.9174311926605505,
+      "grad_norm": 0.9887730479240417,
+      "learning_rate": 0.00015762210099555803,
+      "loss": 0.4526,
+      "num_input_tokens_seen": 59952,
+      "step": 75
+    },
+    {
+      "epoch": 0.9785932721712538,
+      "grad_norm": 0.9649442434310913,
+      "learning_rate": 0.00015222229563517385,
+      "loss": 0.4048,
+      "num_input_tokens_seen": 63840,
+      "step": 80
+    },
+    {
+      "epoch": 1.0489296636085628,
+      "grad_norm": 0.8496802449226379,
+      "learning_rate": 0.0001466043519702539,
+      "loss": 0.3599,
+      "num_input_tokens_seen": 68176,
+      "step": 85
+    },
+    {
+      "epoch": 1.110091743119266,
+      "grad_norm": 0.6206537485122681,
+      "learning_rate": 0.00014079173677281837,
+      "loss": 0.2197,
+      "num_input_tokens_seen": 72336,
+      "step": 90
+    },
+    {
+      "epoch": 1.1712538226299694,
+      "grad_norm": 0.7237979173660278,
+      "learning_rate": 0.00013480872997940905,
+      "loss": 0.1816,
+      "num_input_tokens_seen": 76384,
+      "step": 95
+    },
+    {
+      "epoch": 1.2324159021406729,
+      "grad_norm": 1.0752713680267334,
+      "learning_rate": 0.00012868032327110904,
+      "loss": 0.1839,
+      "num_input_tokens_seen": 80368,
+      "step": 100
+    },
+    {
+      "epoch": 1.2935779816513762,
+      "grad_norm": 1.0526070594787598,
+      "learning_rate": 0.00012243211568052677,
+      "loss": 0.21,
+      "num_input_tokens_seen": 84400,
+      "step": 105
+    },
+    {
+      "epoch": 1.3547400611620795,
+      "grad_norm": 1.073427677154541,
+      "learning_rate": 0.00011609020666180575,
+      "loss": 0.2568,
+      "num_input_tokens_seen": 88352,
+      "step": 110
+    },
+    {
+      "epoch": 1.4159021406727827,
+      "grad_norm": 0.8212242722511292,
+      "learning_rate": 0.00010968108707031792,
+      "loss": 0.2049,
+      "num_input_tokens_seen": 92464,
+      "step": 115
+    },
+    {
+      "epoch": 1.4770642201834863,
+      "grad_norm": 1.1588611602783203,
+      "learning_rate": 0.00010323152850743107,
+      "loss": 0.2213,
+      "num_input_tokens_seen": 96432,
+      "step": 120
+    },
+    {
+      "epoch": 1.5382262996941896,
+      "grad_norm": 1.08698308467865,
+      "learning_rate": 9.676847149256895e-05,
+      "loss": 0.2836,
+      "num_input_tokens_seen": 100416,
+      "step": 125
+    },
+    {
+      "epoch": 1.599388379204893,
+      "grad_norm": 1.1150074005126953,
+      "learning_rate": 9.03189129296821e-05,
+      "loss": 0.228,
+      "num_input_tokens_seen": 104432,
+      "step": 130
+    },
+    {
+      "epoch": 1.6605504587155964,
+      "grad_norm": 0.7235325574874878,
+      "learning_rate": 8.390979333819426e-05,
+      "loss": 0.1778,
+      "num_input_tokens_seen": 108432,
+      "step": 135
+    },
+    {
+      "epoch": 1.7217125382262997,
+      "grad_norm": 0.8855745196342468,
+      "learning_rate": 7.756788431947326e-05,
+      "loss": 0.1871,
+      "num_input_tokens_seen": 112320,
+      "step": 140
+    },
+    {
+      "epoch": 1.782874617737003,
+      "grad_norm": 1.3950036764144897,
+      "learning_rate": 7.131967672889101e-05,
+      "loss": 0.2289,
+      "num_input_tokens_seen": 116272,
+      "step": 145
+    },
+    {
+      "epoch": 1.8440366972477065,
+      "grad_norm": 1.182417392730713,
+      "learning_rate": 6.519127002059095e-05,
+      "loss": 0.2889,
+      "num_input_tokens_seen": 120224,
+      "step": 150
+    },
+    {
+      "epoch": 1.9051987767584098,
+      "grad_norm": 1.020156741142273,
+      "learning_rate": 5.920826322718165e-05,
+      "loss": 0.2724,
+      "num_input_tokens_seen": 124176,
+      "step": 155
+    },
+    {
+      "epoch": 1.9663608562691133,
+      "grad_norm": 0.8830191493034363,
+      "learning_rate": 5.339564802974615e-05,
+      "loss": 0.2359,
+      "num_input_tokens_seen": 128256,
+      "step": 160
+    },
+    {
+      "epoch": 2.036697247706422,
+      "grad_norm": 0.8336369395256042,
+      "learning_rate": 4.777770436482617e-05,
+      "loss": 0.1864,
+      "num_input_tokens_seen": 132576,
+      "step": 165
+    },
+    {
+      "epoch": 2.0978593272171255,
+      "grad_norm": 0.583479106426239,
+      "learning_rate": 4.2377899004441966e-05,
+      "loss": 0.0671,
+      "num_input_tokens_seen": 136608,
+      "step": 170
+    },
+    {
+      "epoch": 2.159021406727829,
+      "grad_norm": 0.686152994632721,
+      "learning_rate": 3.721878753279017e-05,
+      "loss": 0.1465,
+      "num_input_tokens_seen": 140416,
+      "step": 175
+    },
+    {
+      "epoch": 2.220183486238532,
+      "grad_norm": 0.6811323761940002,
+      "learning_rate": 3.2321920129073816e-05,
+      "loss": 0.091,
+      "num_input_tokens_seen": 144336,
+      "step": 180
+    },
+    {
+      "epoch": 2.2813455657492354,
+      "grad_norm": 0.5875197052955627,
+      "learning_rate": 2.770775155002071e-05,
+      "loss": 0.114,
+      "num_input_tokens_seen": 148256,
+      "step": 185
+    },
+    {
+      "epoch": 2.3425076452599387,
+      "grad_norm": 0.6630216836929321,
+      "learning_rate": 2.339555568810221e-05,
+      "loss": 0.1381,
+      "num_input_tokens_seen": 152416,
+      "step": 190
+    },
+    {
+      "epoch": 2.4036697247706424,
+      "grad_norm": 0.7730478644371033,
+      "learning_rate": 1.9403345062352573e-05,
+      "loss": 0.0799,
+      "num_input_tokens_seen": 156400,
+      "step": 195
+    },
+    {
+      "epoch": 2.4648318042813457,
+      "grad_norm": 0.7855842709541321,
+      "learning_rate": 1.5747795578085046e-05,
+      "loss": 0.0722,
+      "num_input_tokens_seen": 160528,
+      "step": 200
+    },
+    {
+      "epoch": 2.525993883792049,
+      "grad_norm": 0.7239196300506592,
+      "learning_rate": 1.2444176869790925e-05,
+      "loss": 0.1003,
+      "num_input_tokens_seen": 164720,
+      "step": 205
+    },
+    {
+      "epoch": 2.5871559633027523,
+      "grad_norm": 0.8815809488296509,
+      "learning_rate": 9.506288518187467e-06,
+      "loss": 0.1155,
+      "num_input_tokens_seen": 168560,
+      "step": 210
+    },
+    {
+      "epoch": 2.6483180428134556,
+      "grad_norm": 1.0388275384902954,
+      "learning_rate": 6.946402407843155e-06,
+      "loss": 0.0962,
+      "num_input_tokens_seen": 172608,
+      "step": 215
+    },
+    {
+      "epoch": 2.709480122324159,
+      "grad_norm": 0.9285043478012085,
+      "learning_rate": 4.775211466158469e-06,
+      "loss": 0.0808,
+      "num_input_tokens_seen": 176656,
+      "step": 220
+    },
+    {
+      "epoch": 2.770642201834862,
+      "grad_norm": 0.9785643815994263,
+      "learning_rate": 3.0017849978256516e-06,
+      "loss": 0.1206,
+      "num_input_tokens_seen": 180608,
+      "step": 225
+    },
+    {
+      "epoch": 2.8318042813455655,
+      "grad_norm": 0.9847484827041626,
+      "learning_rate": 1.6335308013398886e-06,
+      "loss": 0.1259,
+      "num_input_tokens_seen": 184592,
+      "step": 230
+    },
+    {
+      "epoch": 2.8929663608562692,
+      "grad_norm": 1.2911163568496704,
+      "learning_rate": 6.761642258056978e-07,
+      "loss": 0.158,
+      "num_input_tokens_seen": 188496,
+      "step": 235
+    },
+    {
+      "epoch": 2.9541284403669725,
+      "grad_norm": 0.7206448912620544,
+      "learning_rate": 1.3368429729168076e-07,
+      "loss": 0.062,
+      "num_input_tokens_seen": 192368,
+      "step": 240
+    },
+    {
+      "epoch": 2.9908256880733948,
+      "num_input_tokens_seen": 194864,
+      "step": 243,
+      "total_flos": 7611007265931264.0,
+      "train_loss": 0.34003226710445106,
+      "train_runtime": 5382.7216,
+      "train_samples_per_second": 0.728,
+      "train_steps_per_second": 0.045
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 243,
+  "num_input_tokens_seen": 194864,
+  "num_train_epochs": 3,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7611007265931264.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e398eee9bde032e96ecf2e3858ff52edd7fac6c2282b2292935f8e43f27802b3
+size 5752

training_args.yaml

@@ -0,0 +1,36 @@
+bf16: true
+cutoff_len: 256
+dataset: alpaca_en_demo
+dataset_dir: data
+ddp_timeout: 180000000
+do_train: true
+double_quantization: true
+finetuning_type: lora
+flash_attn: auto
+gradient_accumulation_steps: 8
+include_num_input_tokens_seen: true
+learning_rate: 0.0002
+logging_steps: 5
+lora_alpha: 16
+lora_dropout: 0
+lora_rank: 8
+lora_target: all
+lr_scheduler_type: cosine
+max_grad_norm: 0.3
+max_samples: 10000
+model_name_or_path: deepseek-ai/deepseek-llm-7b-base
+num_train_epochs: 3.0
+optim: adamw_torch
+output_dir: saves/DeepSeek-LLM-7B-Base/lora/train_2025-04-22-23-24-06
+packing: false
+per_device_train_batch_size: 2
+plot_loss: true
+preprocessing_num_workers: 16
+quantization_bit: 4
+quantization_method: bnb
+report_to: none
+save_steps: 100
+stage: sft
+template: default
+trust_remote_code: true
+warmup_steps: 0