Save model at global step 40

config.yaml

@@ -0,0 +1,176 @@
+data:
+  tokenizer: null
+  train_files: dapo_ds_train_sample.parquet
+  val_files: matheval.parquet
+  prompt_key: prompt
+  reward_fn_key: data_source
+  max_prompt_length: 1024
+  max_response_length: 2048
+  train_batch_size: 128
+  val_batch_size: 640
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle: true
+  filter_overlong_prompts: false
+  filter_overlong_prompts_workers: 1
+  truncation: left
+  image_key: images
+  custom_cls:
+    path: null
+    name: null
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: Qwen/Qwen2.5-Math-1.5B
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: false
+    use_liger: false
+    save_hf_repo_id: RyanYr/brm-dapo-qwen2.5math-1.5B-base-lr2.5e-6-beta0.002
+    tokenizer_chat_template: null
+  actor:
+    brm:
+      norm_factor: value
+      value_constant: null
+    buffer:
+      buffer_type: null
+      offline_dataset_buffer:
+        train_files: null
+        response_key: response
+        response_truncation: right
+        shuffle: true
+        update_size: 128
+    strategy: fsdp
+    ppo_mini_batch_size: 128
+    ppo_micro_batch_size: null
+    ppo_micro_batch_size_per_gpu: 4
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    use_torch_compile: true
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    checkpoint:
+      contents:
+      - model
+      - optimizer
+      - extra
+    optim:
+      lr: 2.5e-06
+      lr_warmup_steps: -1
+      lr_warmup_steps_ratio: 0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: 810
+      weight_decay: 0.01
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: false
+      optimizer_offload: false
+      fsdp_size: -1
+  ref:
+    ref_model_path: Qwen/Qwen2.5-Math-1.5B
+    strategy: fsdp
+    fsdp_config:
+      param_offload: false
+      wrap_policy:
+        min_num_params: 0
+    log_prob_micro_batch_size: null
+    log_prob_micro_batch_size_per_gpu: 16
+    log_prob_use_dynamic_bsz: false
+    log_prob_max_token_len_per_gpu: 16384
+    ulysses_sequence_parallel_size: 1
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 1
+    use_fire_sampling: false
+    prompt_length: 1024
+    response_length: 2048
+    dtype: bfloat16
+    gpu_memory_utilization: 0.7
+    ignore_eos: false
+    enforce_eager: false
+    free_cache_engine: false
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 4
+    max_num_batched_tokens: 4072
+    max_model_len: null
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: null
+    log_prob_micro_batch_size_per_gpu: 16
+    log_prob_use_dynamic_bsz: false
+    log_prob_max_token_len_per_gpu: 16384
+    disable_log_stats: true
+    enable_chunked_prefill: true
+    do_sample: true
+    'n': 1
+    engine_kwargs:
+      swap_space: null
+    val_kwargs:
+      top_k: -1
+      top_p: 1.0
+      temperature: 0.6
+      'n': 32
+      do_sample: true
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: Qwen/Qwen2.5-Math-1.5B
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: null
+    use_remove_padding: false
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: false
+      fsdp_size: -1
+  micro_batch_size: null
+  micro_batch_size_per_gpu: null
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: false
+  forward_max_token_len_per_gpu: 16384
+  reward_manager: prime
+  reward_kwargs:
+    format_reward: 0.0
+    format_type: null
+custom_reward_function:
+  path: null
+  name: compute_score
+trainer:
+  balance_batch: true
+  total_epochs: 6
+  total_training_steps: null
+  project_name: value-LLM
+  experiment_name: brm-dapo-qwen2.5math-1.5B-base_lr2.5e-6-beta0.002
+  logger:
+  - console
+  - wandb
+  log_val_generations: 0
+  nnodes: 1
+  n_gpus_per_node: 4
+  save_freq: 40
+  resume_mode: auto
+  resume_from_path: null
+  val_before_train: false
+  test_freq: -1
+  default_hdfs_dir: null
+  del_local_ckpt_after_load: false
+  default_local_dir: ./BRM
+  max_actor_ckpt_to_keep: 1
+  ray_wait_register_center_timeout: 300
+  hf_token: null
+  resume_from_hf:
+    enable: false
+    hf_repo_id: null
+    hf_token: null
+    revision: main
+algorithm:
+  kl_ctrl:
+    kl_coef: 0.002

data.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:972a5213f98845a35d97265f0f211d784de8b77d1d8f876c32d83a5f01de0bda
+size 1492

extra_state_world_size_4_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac478195e64c5adc1589f984ee9504b31ee50d7781d99cdf22ecfe513641e38
+size 14632

extra_state_world_size_4_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd9eb602b937ae3abfdc710b6d8ee83926d7c65b5de1c6c28b9ae4f7e9b142f
+size 14632

extra_state_world_size_4_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80a362d156678550afad0428cd91309dea2ab23369a46041c12e2ed80ad62ff7
+size 14632

extra_state_world_size_4_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a53baac35ae90ef7df9bc4c96819f371f332549cb2b330d17683f70d526a426
+size 14632

model_world_size_4_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:86251f89de498ee633c346e25b3f2960784397b63c13f10781dcb1d81a26b2d5
+size 1777276538

model_world_size_4_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e604abd3287779308deabdba0b53571da05d37f7a755885a95f35f6864fd2ab
+size 1777276538

model_world_size_4_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08940ed45aa2c3fe769b6a93237a2dfd7d8d150842c4487fa3d3045d6646a3d6
+size 1777276538

model_world_size_4_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f0ea4d52c6d12df821bb1426e78a8398a8b375f038a5a21f549b31c5d56d6d4
+size 1777276538

optim_world_size_4_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d3d908e0d8ce6376de568c16933f0fb14a12b723d7836198c4daad383b0b247
+size 3087454775

optim_world_size_4_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dc2ee8e96825a91b3aa645f9063421767c3bdfc466ac4e3cd1fbab8d83a7436
+size 3087454775

optim_world_size_4_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:273b5aebe47cf5198ba6b1cab4e3d5e274aa2497edf13a0c5d02904198b4659e
+size 3087454775

optim_world_size_4_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13add9ec1501920902f6a93cdcc2516715946a2721d0234e1765c973c88013b0
+size 3087454775