Save model at global step 5

config.yaml

@@ -0,0 +1,224 @@
+data:
+  tokenizer: null
+  train_files: dapo_ds_train_sample.parquet
+  val_files: matheval.parquet
+  prompt_key: prompt
+  reward_fn_key: data_source
+  max_prompt_length: 768
+  max_response_length: 13312
+  train_batch_size: 1024
+  val_batch_size: null
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle: true
+  filter_overlong_prompts: true
+  filter_overlong_prompts_workers: 1
+  truncation: error
+  image_key: images
+  custom_cls:
+    path: null
+    name: null
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: true
+    use_liger: false
+    save_hf_repo_id: RyanYr/ppo-dapo-r1qwen1.5B-base-lr-mbs256-gpu8_actor
+    tokenizer_chat_template: null
+  actor:
+    strategy: fsdp
+    ppo_mini_batch_size: 256
+    ppo_micro_batch_size: null
+    ppo_micro_batch_size_per_gpu: 1
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    clip_ratio: 0.2
+    clip_ratio_low: 0.2
+    clip_ratio_high: 0.2
+    clip_ratio_c: 3.0
+    loss_agg_mode: token-mean
+    entropy_coeff: 0.001
+    use_kl_loss: false
+    use_torch_compile: true
+    kl_loss_coef: 0.001
+    kl_loss_type: low_var_kl
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    checkpoint:
+      contents:
+      - model
+      - optimizer
+      - extra
+    optim:
+      lr: 1.0e-06
+      lr_warmup_steps: -1
+      lr_warmup_steps_ratio: 0.0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: 100
+      weight_decay: 0.01
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: false
+      optimizer_offload: false
+      fsdp_size: -1
+  ref:
+    strategy: fsdp
+    fsdp_config:
+      param_offload: false
+      wrap_policy:
+        min_num_params: 0
+    log_prob_micro_batch_size: null
+    log_prob_micro_batch_size_per_gpu: 1
+    log_prob_use_dynamic_bsz: false
+    log_prob_max_token_len_per_gpu: 16384
+    ulysses_sequence_parallel_size: 1
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 1
+    use_fire_sampling: false
+    prompt_length: 768
+    response_length: 13312
+    dtype: bfloat16
+    gpu_memory_utilization: 0.7
+    ignore_eos: false
+    enforce_eager: false
+    free_cache_engine: false
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 4
+    max_num_batched_tokens: 14080
+    max_model_len: null
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: null
+    log_prob_micro_batch_size_per_gpu: 1
+    log_prob_use_dynamic_bsz: false
+    log_prob_max_token_len_per_gpu: 16384
+    disable_log_stats: true
+    enable_chunked_prefill: true
+    do_sample: true
+    'n': 1
+    engine_kwargs:
+      swap_space: null
+    val_kwargs:
+      top_k: -1
+      top_p: 1.0
+      temperature: 0
+      'n': 1
+      do_sample: false
+critic:
+  rollout_n: 1
+  strategy: fsdp
+  optim:
+    lr: 1.0e-05
+    lr_warmup_steps_ratio: 0.0
+    min_lr_ratio: null
+    warmup_style: constant
+    total_training_steps: 100
+    weight_decay: 0.01
+  model:
+    path: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+    tokenizer_path: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+    override_config: {}
+    external_lib: null
+    enable_gradient_checkpointing: false
+    use_remove_padding: true
+    fsdp_config:
+      param_offload: false
+      optimizer_offload: false
+      wrap_policy:
+        min_num_params: 0
+      fsdp_size: -1
+    save_hf_repo_id: RyanYr/ppo-dapo-r1qwen1.5B-base-lr-mbs256-gpu8_critic
+  ppo_mini_batch_size: 256
+  ppo_micro_batch_size: null
+  ppo_micro_batch_size_per_gpu: 1
+  forward_micro_batch_size: null
+  forward_micro_batch_size_per_gpu: 1
+  use_dynamic_bsz: false
+  ppo_max_token_len_per_gpu: 32768
+  forward_max_token_len_per_gpu: 32768
+  ulysses_sequence_parallel_size: 1
+  ppo_epochs: 1
+  shuffle: false
+  grad_clip: 1.0
+  cliprange_value: 0.5
+  checkpoint:
+    contents:
+    - model
+    - optimizer
+    - extra
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: null
+    use_remove_padding: false
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: false
+      fsdp_size: -1
+  micro_batch_size: null
+  micro_batch_size_per_gpu: null
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: false
+  forward_max_token_len_per_gpu: 32768
+  reward_manager: prime
+custom_reward_function:
+  path: null
+  name: compute_score
+algorithm:
+  gamma: 1.0
+  lam: 1.0
+  adv_estimator: gae
+  use_kl_in_reward: true
+  kl_penalty: kl
+  kl_ctrl:
+    type: fixed
+    kl_coef: 0.001
+    horizon: 10000
+    target_kl: 0.1
+trainer:
+  balance_batch: true
+  total_epochs: 100
+  total_training_steps: 100
+  project_name: value-LLM
+  experiment_name: ppo-dapo-r1qwen1.5B-base_lr-mbs256-gpu8
+  logger:
+  - console
+  - wandb
+  log_val_generations: 0
+  nnodes: 1
+  n_gpus_per_node: 8
+  save_freq: 5
+  resume_mode: auto
+  resume_from_path: null
+  val_before_train: false
+  test_freq: -1
+  critic_warmup: 0
+  default_hdfs_dir: null
+  del_local_ckpt_after_load: false
+  default_local_dir: checkpoints/value-LLM/ppo-dapo-r1qwen1.5B-base_lr-mbs256-gpu8
+  max_actor_ckpt_to_keep: 1
+  max_critic_ckpt_to_keep: 1
+  ray_wait_register_center_timeout: 300
+  hf_token: null
+  resume_from_hf:
+    enable: false
+    actor_hf_repo_id: null
+    actor_revision: main
+    critic_hf_repo_id: null
+    critic_revision: main
+    hf_token: null

data.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d652b8fb542c50c9e3da045c04599fc560c10edb7716089c3b430f1f3362e34b
+size 1492

extra_state_world_size_8_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d26a59f27bd732e4c612d8d181bdaf8e2602741fd9168b63b083900414555da6
+size 14632

extra_state_world_size_8_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79ecdb326599a4e54468a177285efcb643176d661ce2ac2e596ee040050c9658
+size 14632

extra_state_world_size_8_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d857c457b5764c683c29ceb5624a113f371f9fc5349661055d4c9e5c13e01dd3
+size 14632

extra_state_world_size_8_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d9eaf0ea4780cb43049910655bd7ed15f8b5eeaa276bcc5f338206b31929425
+size 14632

extra_state_world_size_8_rank_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02e33b60788c672801ab2d3a3532acca8eea49b3d85e814de689602b0156bfa4
+size 14632

extra_state_world_size_8_rank_5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2877d1139c733db5a39689cf05cda32bd5ab88e45fb65d8adcb19242c2807926
+size 14632

extra_state_world_size_8_rank_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75048502776790ad78c23cb22300d7c6ee395ccadc2d9d7ecc40c26972e8ffe6
+size 14632

extra_state_world_size_8_rank_7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44cb2df6c1bef3e6f59919fed64aa5fb97cad895d2aaa0a5e8136e04bc9e4692
+size 14632

model_world_size_8_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23c6641e2e3597b8c44a87570271db2cf2992706461657ae1cded60cc93ac647
+size 888732218

model_world_size_8_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b58da916a79788367114c5e73d4526ec953f64a8cf94d7044b2e2e3c861f1933
+size 888732218

model_world_size_8_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cbe831687b13b9b33ba2b7b2cf10ea76ddcd06c849ca4ae6854f311852e31293
+size 888732218

model_world_size_8_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2c2b37efe3989f2d00a052b97ee6c3e2ce3354ff9583e7d4164093f7f1f3724
+size 888732218

model_world_size_8_rank_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ae5a32bb7a0c03215bf9e617c1984a4c5aeb3e22537158890212a745515df0
+size 888732218

model_world_size_8_rank_5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fe66fbab0c90cffd625840082537b5512d67a9294c783716f40913b8509e724
+size 888732218

model_world_size_8_rank_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b594a7dc92622e3607e82aa2b1d9dfeeb5c56bf48143b37d40200a7d6147568
+size 888732218

model_world_size_8_rank_7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:469709046db106cc227115d79bb9f396e1c5d0b45c94a2058508e03db44ce155
+size 888732218

optim_world_size_8_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce5212d4ab187c6f84c68dbdf93638e25f36dcf2cc74648279f8d467f8a433b4
+size 1777114167

optim_world_size_8_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6702ad776eb9baa73e6ebb6e3cb11fdbc5ccec2203cca48e4069d88e2caabc53
+size 1777114167

optim_world_size_8_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3527368d8b5356f802f65668638e54cbf7b14141456e03a3665f7862bd7c18b9
+size 1777114167

optim_world_size_8_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00e8806c2fe37aea2df5db541edda391410fe765f1153a01705b97bb9a50e6c8
+size 1777114167

optim_world_size_8_rank_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94b56b3a5e919c865020bffbe339fca0d077801b16bf326bb3672ec6d44341f4
+size 1777114167

optim_world_size_8_rank_5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:143df3afc3ed5375791695dfd88a555221cea4dcc27039452b10405b1f9fa757
+size 1777114167

optim_world_size_8_rank_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:80f1e05927b6a2e2f75b76a8836819fa773b643b2c0972f3017d730a20a1beaf
+size 1777114167

optim_world_size_8_rank_7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:524a66d4c035541d58d994f10b12874cfbf0f397e4c6fc0d6c4072db72fb6ba1
+size 1777114167