Save model at global step 40

config.yaml

@@ -5,12 +5,12 @@ data:
   prompt_key: prompt
   reward_fn_key: data_source
   max_prompt_length: 1024
-  max_response_length: 2048
+  max_response_length: 3072
   train_batch_size: 128
   val_batch_size: 640
   return_raw_input_ids: false
   return_raw_chat: false
-  shuffle: true
+  shuffle: false
   filter_overlong_prompts: false
   filter_overlong_prompts_workers: 1
   truncation: left
@@ -39,17 +39,15 @@ actor_rollout_ref:
         train_files: null
         response_key: response
         response_truncation: right
-        shuffle: true
+        shuffle: false
         update_size: 128
     strategy: fsdp
-    ppo_mini_batch_size: 128
-    ppo_micro_batch_size: null
-    ppo_micro_batch_size_per_gpu: 4
+    actor_micro_batch_size: null
+    actor_micro_batch_size_per_gpu: 4
     use_dynamic_bsz: false
-    ppo_max_token_len_per_gpu: 16384
+    actor_max_token_len_per_gpu: 16384
     grad_clip: 1.0
     use_torch_compile: true
-    ppo_epochs: 1
     shuffle: false
     ulysses_sequence_parallel_size: 1
     checkpoint:
@@ -63,7 +61,7 @@ actor_rollout_ref:
       lr_warmup_steps_ratio: 0
       min_lr_ratio: null
       warmup_style: constant
-      total_training_steps: 810
+      total_training_steps: 200
       weight_decay: 0.01
     fsdp_config:
       wrap_policy:
@@ -71,6 +69,10 @@ actor_rollout_ref:
       param_offload: false
       optimizer_offload: false
       fsdp_size: -1
+    report_entropy: true
+    actor_mini_batch_size: 128
+    use_kl_loss: false
+    actor_epochs: 1
   ref:
     ref_model_path: Qwen/Qwen2.5-Math-1.5B
     strategy: fsdp
@@ -90,7 +92,7 @@ actor_rollout_ref:
     top_p: 1
     use_fire_sampling: false
     prompt_length: 1024
-    response_length: 2048
+    response_length: 3072
     dtype: bfloat16
     gpu_memory_utilization: 0.7
     ignore_eos: false
@@ -98,7 +100,7 @@ actor_rollout_ref:
     free_cache_engine: false
     load_format: dummy_dtensor
     tensor_model_parallel_size: 4
-    max_num_batched_tokens: 4072
+    max_num_batched_tokens: 5096
     max_model_len: null
     max_num_seqs: 1024
     log_prob_micro_batch_size: null
@@ -130,8 +132,8 @@ reward_model:
         min_num_params: 0
       param_offload: false
       fsdp_size: -1
-  micro_batch_size: null
-  micro_batch_size_per_gpu: null
+  actor_micro_batch_size: null
+  actor_micro_batch_size_per_gpu: null
   max_length: null
   ulysses_sequence_parallel_size: 1
   use_dynamic_bsz: false
@@ -145,8 +147,8 @@ custom_reward_function:
   name: compute_score
 trainer:
   balance_batch: true
-  total_epochs: 6
-  total_training_steps: null
+  total_epochs: 100
+  total_training_steps: 200
   project_name: value-LLM
   experiment_name: brm-dapo-qwen2.5math-1.5B-base_lr2.5e-6-beta0.002
   logger:
@@ -174,3 +176,5 @@ trainer:
 algorithm:
   kl_ctrl:
     kl_coef: 0.002
+  use_kl_in_reward: false
+  adv_estimator: none

data.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0f6f475faf0bfb0b063d0aea47322ec9932da50a02ee37ea47d0d0b62293632c
+oid sha256:1c212e96abb1f5ae386cf5dc23176f00a5736862617b92bf3de966c5dfda9400
 size 1492

extra_state_world_size_4_rank_0.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17728ff021915ac36c35065b209e46a3a9dc992206150437a91c93859fd77ba5
+oid sha256:bac478195e64c5adc1589f984ee9504b31ee50d7781d99cdf22ecfe513641e38
 size 14632

extra_state_world_size_4_rank_1.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f99cb3c92a09a0d226b1ebd90e9be63e77702416878b63404992f60b94d8491e
+oid sha256:5bd9eb602b937ae3abfdc710b6d8ee83926d7c65b5de1c6c28b9ae4f7e9b142f
 size 14632

extra_state_world_size_4_rank_2.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6bea50df638fba3dbf7ba2c4472acc8a4a6bbe82742465214401b73208e3caac
+oid sha256:80a362d156678550afad0428cd91309dea2ab23369a46041c12e2ed80ad62ff7
 size 14632

extra_state_world_size_4_rank_3.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1bf91e43636e0ea8ba3dd299748b9997896cd326ac699e730584f1c572af319
+oid sha256:4a53baac35ae90ef7df9bc4c96819f371f332549cb2b330d17683f70d526a426
 size 14632

model_world_size_4_rank_0.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:064a8fa7f15738276e0011ddfdad872f733cc9a3b31f2bde587079e8079bd3a9
+oid sha256:056770c9e91a4a9083cebcbd610e2db8050a57faa7f76c145bfee83d3da0c9e4
 size 1777276538

model_world_size_4_rank_1.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18693386a1bf50f2f6d91aa95df67c9beb1e2b00f738ffe2b129da78ee7ddb7f
+oid sha256:ffc8cace4da138ff67214cffba51b8443661fd41f4079d762ee0f2310a38f3b9
 size 1777276538

model_world_size_4_rank_2.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2b0de637e4c0caa8a6ab883067552b98673e979b7d6f96d2dfd0cd1b520f7376
+oid sha256:089b2235d15061977973a73ffd2f75863dd4c83294873ee4c0c578b13534a6d5
 size 1777276538

model_world_size_4_rank_3.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa0e98eb308fe6d82c6078cfaf4019a3f81f0e0b496b4ded3d050fdb7a751751
+oid sha256:b5ada2141949e97f0b6b4248ef91506dbfe959a5f6e42671c3bde140e23d891e
 size 1777276538

optim_world_size_4_rank_0.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:014c6d547b159c558d0ac0aa24ae0d183acd34412b8135c583786778bc816d0a
+oid sha256:7373adeb1aff488c5e1f4f9ff6065314aff9b175eb58166fba53d03c643b45dd
 size 3087454775

optim_world_size_4_rank_1.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:936d0ba5343dd68dbec1bf66554d39e8c377c507af4c34380a2885b5ab375092
+oid sha256:8ea11f1272e9b4971d23bf2e84177de0127c00396539e9bce57d53b7033730c0
 size 3087454775

optim_world_size_4_rank_2.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83991b0da5eb38438d33936780c81a81e0312f147ec7ed759790eaa0f698b2bb
+oid sha256:5a61c4cdf03fd68426e3a3799500a5e729e3616c97becfbcab00d0fe618f3fd3
 size 3087454775

optim_world_size_4_rank_3.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:25cfed09ceaa90e51e88dbd007b479bf35fb3509caca301b0468b6e3d3a7ddc5
+oid sha256:7bc903b6b259fd4f43e397b943be707c2ef26d94ace7a99fe322360e84c6a262
 size 3087454775