Training in progress, step 40

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

00000000-0000-0000-0000-000000000000.yml

@@ -0,0 +1,50 @@
+base_model: peft-internal-testing/tiny-dummy-qwen2
+batch_size: 8
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- format: custom
+  path: argilla/databricks-dolly-15k-curated-en
+  type:
+    field_input: original-instruction
+    field_instruction: original-instruction
+    field_output: original-response
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/test-repo
+hub_strategy: checkpoint
+learning_rate: 0.002
+load_best_model_at_end: true
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 1
+micro_batch_size: 1
+model_type: AutoModelForCausalLM
+num_epochs: 100
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 8
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.001
+wandb_entity: ''
+wandb_mode: online
+wandb_name: peft-internal-testing/tiny-dummy-qwen2-argilla/databricks-dolly-15k-curated-en
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

01b60291-41f3-4631-b7e8-f7c60c2ca163.yml

@@ -0,0 +1,53 @@
+base_model: huggyllama/llama-7b
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - ccd32583f980ebf0_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/ccd32583f980ebf0_train_data.json
+  type:
+    field_input: ''
+    field_instruction: problem
+    field_output: solution
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/fa36bf4c-34a6-4e51-ae14-a8372bf92b39
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: </s>
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: huggyllama/llama-7b-/workspace/input_data/ccd32583f980ebf0_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

03a659ff-e350-4bb9-8ff3-8c658a5d0dff.yml

@@ -0,0 +1,52 @@
+base_model: fxmarty/tiny-llama-fast-tokenizer
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - fc6136aac03f618a_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/fc6136aac03f618a_train_data.json
+  type:
+    field_instruction: text
+    field_output: title
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/b1c9c4ec-ffa2-429d-9c5b-90b5979c502d
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: </s>
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: fxmarty/tiny-llama-fast-tokenizer-/workspace/input_data/fc6136aac03f618a_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

0573e27c-6a6f-4989-9b30-ba29f333396d.yml

@@ -0,0 +1,50 @@
+base_model: dltjdgh0928/test_instruction
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 76022a30315552b8_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/76022a30315552b8_train_data.json
+  type:
+    field_instruction: input
+    field_output: target
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/c32e2a6e-d46b-44d8-953f-917301378acd
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: dltjdgh0928/test_instruction-/workspace/input_data/76022a30315552b8_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

077fd330-87f9-4bc4-b449-7713fbdaf1b0.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/mistral-7b-v0.3
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - ca0152973425c947_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/ca0152973425c947_train_data.json
+  type:
+    field_input: code
+    field_instruction: func_name
+    field_output: docstring
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/5a2f5ce6-446b-4282-bb4d-9ee4e970231f
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/mistral-7b-v0.3-/tmp/ca0152973425c947_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

1dc178e8-8f66-48ae-8ebb-825428c168d0.yml

@@ -0,0 +1,53 @@
+base_model: NousResearch/Yarn-Mistral-7b-64k
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - bccab6bcbcb6fc03_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/bccab6bcbcb6fc03_train_data.json
+  type:
+    field_input: choices
+    field_instruction: full_prompt
+    field_output: example
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/93085afc-6d0b-49ca-ac4a-839ea57462a9
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: </s>
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: NousResearch/Yarn-Mistral-7b-64k-/workspace/input_data/bccab6bcbcb6fc03_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

21315ae5-16ee-43cd-9612-743524060933.yml

@@ -0,0 +1,50 @@
+base_model: unsloth/Meta-Llama-3.1-8B
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 562fa3aeea07046a_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/562fa3aeea07046a_train_data.json
+  type:
+    field_instruction: prompt
+    field_output: text
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/c4596edc-efad-4776-86a1-caa06bffcada
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: PreTrainedTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Meta-Llama-3.1-8B-/workspace/input_data/562fa3aeea07046a_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

284c3982-7bc3-4e42-a78c-849f03798c5f.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/Qwen2-7B-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 3e306f9221b79797_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/3e306f9221b79797_train_data.json
+  type:
+    field_input: dialogue
+    field_instruction: rendered_input
+    field_output: summary
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/f3c9c6cc-5806-45b4-aab9-d03de6022b3a
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Qwen2-7B-Instruct-/workspace/input_data/3e306f9221b79797_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

29e89a2c-6136-48b6-88bc-a0066652be7d.yml

@@ -0,0 +1,52 @@
+base_model: JackFram/llama-68m
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - ff3a521d02fa72b2_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/ff3a521d02fa72b2_train_data.json
+  type:
+    field_instruction: context
+    field_output: question
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/4ada8092-cc1e-445c-9260-a580ef2586ae
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: </s>
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: JackFram/llama-68m-/workspace/input_data/ff3a521d02fa72b2_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

2cc30bfb-2df3-4b31-b1fb-e29900be6958.yml

@@ -0,0 +1,53 @@
+base_model: NousResearch/Llama-3.2-1B
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - f51beb4c568b9128_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/f51beb4c568b9128_train_data.json
+  type:
+    field_input: keywords
+    field_instruction: idea
+    field_output: full_response
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/0c2649cc-2fe7-4e88-b672-6da1fee4001f
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: <|end_of_text|>
+tokenizer_type: PreTrainedTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: NousResearch/Llama-3.2-1B-/workspace/input_data/f51beb4c568b9128_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

2eaa630f-7785-4ca3-b46f-be41dcf74f78.yml

@@ -0,0 +1,51 @@
+base_model: katuni4ka/tiny-random-qwen1.5-moe
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 95544452e61c7393_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/95544452e61c7393_train_data.json
+  type:
+    field_input: input
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/e61e89f0-854a-4922-8d25-dae435e91af0
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: katuni4ka/tiny-random-qwen1.5-moe-/workspace/input_data/95544452e61c7393_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

40f27435-f59d-488f-b2d6-01e356d79c48.yml

@@ -0,0 +1,50 @@
+base_model: Qwen/Qwen2-1.5B-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - df925134bb2c32b8_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/df925134bb2c32b8_train_data.json
+  type:
+    field_instruction: prompt
+    field_output: amoral
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/ba640bbe-3257-40d8-88fe-26152f412bb7
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: Qwen/Qwen2-1.5B-Instruct-/tmp/df925134bb2c32b8_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

427d02be-6008-4556-9a5e-9c7cb7503058.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/Phi-3.5-mini-instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 7e5b54272524b996_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/7e5b54272524b996_train_data.json
+  type:
+    field_input: input
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/ae26a9e9-089e-4d4a-b592-d8935df7c18d
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Phi-3.5-mini-instruct-/workspace/input_data/7e5b54272524b996_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

54c39bbc-809b-4c67-a254-0e03a4884b4e.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/codegemma-7b-it
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 2ebe89763cb3150d_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/2ebe89763cb3150d_train_data.json
+  type:
+    field_input: input
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/a0bc38f8-fcd3-4d7e-9a3f-3aa2e8a4204f
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GemmaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/codegemma-7b-it-/tmp/2ebe89763cb3150d_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

5b6d979e-5f1d-47f4-a5d3-c1026b8550e5.yml

@@ -0,0 +1,51 @@
+base_model: princeton-nlp/gemma-2-9b-it-SimPO
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - dbe5c72dde5e5bcb_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/dbe5c72dde5e5bcb_train_data.json
+  type:
+    field_input: essay
+    field_instruction: prompt
+    field_output: evaluation
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/1c0e6767-6440-4997-87d3-2298e513b046
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 2
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GemmaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: princeton-nlp/gemma-2-9b-it-SimPO-/workspace/input_data/dbe5c72dde5e5bcb_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

5ff7bf5f-96dc-43dd-aeeb-560c0ab78db8.yml

@@ -0,0 +1,51 @@
+base_model: NousResearch/Hermes-3-Llama-3.1-8B
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 69447058613b41d8_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/69447058613b41d8_train_data.json
+  type:
+    field_input: sectionParentTitre
+    field_instruction: title_main
+    field_output: texte
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/942aa5fc-b540-46ce-b482-e38c4f637264
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: PreTrainedTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: NousResearch/Hermes-3-Llama-3.1-8B-/workspace/input_data/69447058613b41d8_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

63345f8a-4ec9-47f0-9956-6eaa52b2c2a6.yml

@@ -0,0 +1,52 @@
+base_model: heegyu/WizardVicuna-open-llama-3b-v2
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - fe9267419ea75ad2_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/fe9267419ea75ad2_train_data.json
+  type:
+    field_instruction: ca_topic
+    field_output: article
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/7114c34f-852f-43da-b985-b7f0b6d6d724
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: </s>
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: heegyu/WizardVicuna-open-llama-3b-v2-/tmp/fe9267419ea75ad2_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

6c7ae056-3b4d-460b-ba7b-a4000f32b3f1.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/gemma-2-2b
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - b98d5b59c20c6595_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/b98d5b59c20c6595_train_data.json
+  type:
+    field_input: metadata
+    field_instruction: text
+    field_output: tags_str
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/429ee307-6dd2-4dd7-9e1d-7384d807a3df
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GemmaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/gemma-2-2b-/tmp/b98d5b59c20c6595_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

75b21ca4-feab-4bdd-92b0-ea6d90dfa18f.yml

@@ -0,0 +1,51 @@
+base_model: Qwen/Qwen2.5-1.5B-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - c6adcdcb593a3ee4_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/c6adcdcb593a3ee4_train_data.json
+  type:
+    field_input: abstract
+    field_instruction: question_en_origin
+    field_output: answer_en_origin
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/2faf844e-4a0a-4d23-95f4-a055e4864133
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: Qwen/Qwen2.5-1.5B-Instruct-/workspace/input_data/c6adcdcb593a3ee4_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

7a4a0d08-b201-4939-999e-8cad606c5cdd.yml

@@ -0,0 +1,50 @@
+base_model: NousResearch/GPT4-x-Vicuna-13b-fp16
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - dcf32f9d35bdd1f9_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/dcf32f9d35bdd1f9_train_data.json
+  type:
+    field_instruction: doc_text
+    field_output: summary_text
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/009ef170-2771-4ab8-8e1b-9a9d2a2e1e2b
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 2
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: NousResearch/GPT4-x-Vicuna-13b-fp16-/workspace/input_data/dcf32f9d35bdd1f9_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

879db250-c3f5-4d43-a7c5-c5a456ae5803.yml

@@ -0,0 +1,50 @@
+base_model: unsloth/Qwen2.5-Coder-1.5B-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 4d85b564dafa38db_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/4d85b564dafa38db_train_data.json
+  type:
+    field_instruction: prompt
+    field_output: response
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/876ff803-5357-4240-8766-c54166515403
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Qwen2.5-Coder-1.5B-Instruct-/workspace/input_data/4d85b564dafa38db_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

8910478d-79cf-499e-8fed-7a2142f7ee60.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/Phi-3-medium-4k-instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - f6199f34ade98809_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/f6199f34ade98809_train_data.json
+  type:
+    field_input: choices
+    field_instruction: question
+    field_output: answer
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/2d37ba50-cd70-4895-be62-3477f5193e86
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Phi-3-medium-4k-instruct-/tmp/f6199f34ade98809_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

README.md

@@ -0,0 +1,139 @@
+---
+library_name: transformers
+license: llama3.2
+base_model: NousResearch/Llama-3.2-1B
+tags:
+- axolotl
+- generated_from_trainer
+model-index:
+- name: 0c2649cc-2fe7-4e88-b672-6da1fee4001f
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
+<details><summary>See axolotl config</summary>
+
+axolotl version: `0.4.1`
+```yaml
+base_model: NousResearch/Llama-3.2-1B
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - f51beb4c568b9128_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/f51beb4c568b9128_train_data.json
+  type:
+    field_input: keywords
+    field_instruction: idea
+    field_output: full_response
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/0c2649cc-2fe7-4e88-b672-6da1fee4001f
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: <|end_of_text|>
+tokenizer_type: PreTrainedTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: NousResearch/Llama-3.2-1B-/workspace/input_data/f51beb4c568b9128_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true
+
+```
+
+</details><br>
+
+# 0c2649cc-2fe7-4e88-b672-6da1fee4001f
+
+This model is a fine-tuned version of [NousResearch/Llama-3.2-1B](https://huggingface.co/NousResearch/Llama-3.2-1B) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0849
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 32
+- total_eval_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 12
+- training_steps: 258
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| No log        | 0.0005 | 1    | 0.2074          |
+| 0.5472        | 0.0097 | 20   | 0.1746          |
+| 0.3199        | 0.0194 | 40   | 0.2036          |
+| 0.2013        | 0.0291 | 60   | 0.1772          |
+| 0.0903        | 0.0388 | 80   | 0.1702          |
+| 0.0875        | 0.0485 | 100  | 0.2040          |
+| 0.1425        | 0.0582 | 120  | 0.1392          |
+| 0.1982        | 0.0679 | 140  | 0.1194          |
+| 0.1372        | 0.0776 | 160  | 0.1014          |
+| 0.0278        | 0.0873 | 180  | 0.0952          |
+| 0.0248        | 0.0970 | 200  | 0.0893          |
+| 0.1051        | 0.1067 | 220  | 0.0875          |
+| 0.0649        | 0.1164 | 240  | 0.0849          |
+
+
+### Framework versions
+
+- Transformers 4.46.0
+- Pytorch 2.5.0+cu124
+- Datasets 3.0.1
+- Tokenizers 0.20.1

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

ba646963-47d5-4d28-bb73-74fd1aef7feb.yml

@@ -0,0 +1,50 @@
+base_model: unsloth/Qwen2-0.5B
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 1c60ef1fa1ddd4a9_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/1c60ef1fa1ddd4a9_train_data.json
+  type:
+    field_instruction: en
+    field_output: es
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/054d8bb5-59eb-4c69-9472-ab1b71a92df6
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/Qwen2-0.5B-/workspace/input_data/1c60ef1fa1ddd4a9_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

c6d606c5-1bf1-4d46-8f27-e3893d012d1d.yml

@@ -0,0 +1,50 @@
+base_model: trl-internal-testing/tiny-random-LlamaForCausalLM
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - f4a61305a746447c_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/f4a61305a746447c_train_data.json
+  type:
+    field_instruction: sentence1
+    field_output: sentence2
+    format: '{instruction}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/dab16ec4-4ddf-4ee5-8888-3dc2a83f0f86
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: LlamaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: trl-internal-testing/tiny-random-LlamaForCausalLM-/workspace/input_data/f4a61305a746447c_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

config.json

@@ -0,0 +1,28 @@
+{
+  "_name_or_path": "Qwen/Qwen2-0.5B",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "eos_token_id": 151643,
+  "hidden_act": "silu",
+  "hidden_size": 896,
+  "initializer_range": 0.02,
+  "intermediate_size": 4864,
+  "max_position_embeddings": 131072,
+  "max_window_layers": 24,
+  "model_type": "qwen2",
+  "num_attention_heads": 14,
+  "num_hidden_layers": 24,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

da9e44b3-e4fb-4905-9c7c-6b03aad6b593.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/SmolLM2-360M-Instruct
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - f1ccd02a885008e6_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/f1ccd02a885008e6_train_data.json
+  type:
+    field_input: target
+    field_instruction: user
+    field_output: assistant
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/3da0a03a-adbb-42e3-8fd7-bd7c0b1d3e9f
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GPT2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/SmolLM2-360M-Instruct-/tmp/f1ccd02a885008e6_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

e51d7c64-39d2-4079-b777-8892db299c2a.yml

@@ -0,0 +1,51 @@
+base_model: fxmarty/tiny-random-GemmaForCausalLM
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - b7c2a4a781c93416_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/b7c2a4a781c93416_train_data.json
+  type:
+    field_input: context
+    field_instruction: question
+    field_output: answer
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/fd1980a0-7e71-4e52-addb-318dca5991d5
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GemmaTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: fxmarty/tiny-random-GemmaForCausalLM-/workspace/input_data/b7c2a4a781c93416_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

ee62f35d-1a99-4f1c-a69c-c91bc444b71f.yml

@@ -0,0 +1,53 @@
+base_model: EleutherAI/pythia-1b
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - b2a4966d9a5c880e_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/b2a4966d9a5c880e_train_data.json
+  type:
+    field_input: input
+    field_instruction: instruction
+    field_output: output
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/ee937811-31d0-4e11-944a-f4f8e06309d2
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+special_tokens:
+  pad_token: <|endoftext|>
+tokenizer_type: GPTNeoXTokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: EleutherAI/pythia-1b-/workspace/input_data/b2a4966d9a5c880e_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

ef61f40b-eca8-4670-964b-fdd3d1d0f066.yml

@@ -0,0 +1,51 @@
+base_model: unsloth/SmolLM-135M
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 658988857b0a29c9_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/658988857b0a29c9_train_data.json
+  type:
+    field_input: choices
+    field_instruction: subject
+    field_output: question
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/09370687-f28e-45e5-91f6-f87011850a94
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: GPT2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: unsloth/SmolLM-135M-/workspace/input_data/658988857b0a29c9_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

fb974fc0-f90c-40fd-bd22-46669f42b395.yml

@@ -0,0 +1,51 @@
+base_model: Qwen/Qwen2-0.5B
+batch_size: 32
+bf16: true
+chat_template: tokenizer_default_fallback_alpaca
+datasets:
+- data_files:
+  - 745d2d05aaed18f4_train_data.json
+  ds_type: json
+  format: custom
+  path: /workspace/input_data/745d2d05aaed18f4_train_data.json
+  type:
+    field_input: pos
+    field_instruction: task
+    field_output: query
+    format: '{instruction} {input}'
+    no_input_format: '{instruction}'
+    system_format: '{system}'
+    system_prompt: ''
+eval_steps: 20
+flash_attention: true
+gpu_memory_limit: 80GiB
+gradient_checkpointing: true
+group_by_length: true
+hub_model_id: willtensora/459779f2-cbce-4ec0-b11c-1dcdf92498d8
+hub_strategy: checkpoint
+learning_rate: 0.0002
+logging_steps: 10
+lr_scheduler: cosine
+max_steps: 2500
+micro_batch_size: 4
+model_type: AutoModelForCausalLM
+optimizer: adamw_bnb_8bit
+output_dir: /workspace/axolotl/configs
+pad_to_sequence_len: true
+resize_token_embeddings_to_32x: false
+sample_packing: false
+save_steps: 40
+save_total_limit: 1
+sequence_len: 2048
+tokenizer_type: Qwen2TokenizerFast
+train_on_inputs: false
+trust_remote_code: true
+val_set_size: 0.1
+wandb_entity: ''
+wandb_mode: online
+wandb_name: Qwen/Qwen2-0.5B-/workspace/input_data/745d2d05aaed18f4_train_data.json
+wandb_project: Gradients-On-Demand
+wandb_run: your_name
+wandb_runid: default
+warmup_ratio: 0.05
+xformers_attention: true

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": 128001,
+  "transformers_version": "4.46.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a83931a9a94f9632379d50ef2a7c25f6c054134ce8e68739fdab1e8235bfc9b5
+size 988097824

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a3488e39325dea60c21ab0cf3a2715d26192702fde06183582341380d5a328b
+size 2471678226

special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9
+size 11418266

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null,
+  "use_fast": true
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3bf26f85220d2e1869df4e13339d28deef8b39d73592068f8d91b2a230f2ba47
+size 6648