Model save

README.md

@@ -0,0 +1,72 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: zephyr-7b-UC-0-B0.5
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-UC-0-B0.5
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.1130
+- Rewards/chosen: -3.7732
+- Rewards/rejected: -13.4408
+- Rewards/accuracies: 0.8644
+- Rewards/margins: 9.6676
+- Logps/rejected: -173.6967
+- Logps/chosen: -189.7024
+- Logits/rejected: -2.8645
+- Logits/chosen: -2.9068
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 16
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 0.1262        | 0.62  | 500  | 0.1185          | -3.0114        | -12.1820         | 0.8670             | 9.1707          | -171.1792      | -188.1787    | -2.8565         | -2.9004       |
+
+
+### Framework versions
+
+- Transformers 4.36.1
+- Pytorch 2.0.1+cu117
+- Datasets 2.16.1
+- Tokenizers 0.15.0

all_results.json

@@ -0,0 +1,21 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.9068422317504883,
+    "eval_logits/rejected": -2.864485025405884,
+    "eval_logps/chosen": -189.702392578125,
+    "eval_logps/rejected": -173.6967315673828,
+    "eval_loss": 0.11297444999217987,
+    "eval_rewards/accuracies": 0.8643617033958435,
+    "eval_rewards/chosen": -3.773209571838379,
+    "eval_rewards/margins": 9.667583465576172,
+    "eval_rewards/rejected": -13.440792083740234,
+    "eval_runtime": 242.5683,
+    "eval_samples": 1495,
+    "eval_samples_per_second": 6.163,
+    "eval_steps_per_second": 0.194,
+    "train_loss": 0.14663515942437308,
+    "train_runtime": 4275.5798,
+    "train_samples": 12879,
+    "train_samples_per_second": 3.012,
+    "train_steps_per_second": 0.188
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.36.1",
+  "use_cache": false,
+  "vocab_size": 32000
+}

eval_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 1.0,
+    "eval_logits/chosen": -2.9068422317504883,
+    "eval_logits/rejected": -2.864485025405884,
+    "eval_logps/chosen": -189.702392578125,
+    "eval_logps/rejected": -173.6967315673828,
+    "eval_loss": 0.11297444999217987,
+    "eval_rewards/accuracies": 0.8643617033958435,
+    "eval_rewards/chosen": -3.773209571838379,
+    "eval_rewards/margins": 9.667583465576172,
+    "eval_rewards/rejected": -13.440792083740234,
+    "eval_runtime": 242.5683,
+    "eval_samples": 1495,
+    "eval_samples_per_second": 6.163,
+    "eval_steps_per_second": 0.194
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.36.1"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:050e6c6e157d883999fbd95f3f5bfe20c8ff7fe3bfb8520bf7de15319e5f308b
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d94280de7f96a6d0838668de2e1fc5c24f5380093f07da0bde5cca354dadbf8c
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd27b1c21675637c170c1e0d747e1597faef82ff3ce5a8208f09198cfb00d0e0
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Feb06_09-53-23_node08/events.out.tfevents.1707184485.node08.2968855.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8cb1d5805b72cc4f68d61755bc5bc0db1950e82e5088b59a0b4984e7b9a49453
+size 8187

runs/Feb06_11-48-54_node01/events.out.tfevents.1707191358.node01.4168700.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b21b953b41ad4ec09822823951476f6616e1cac1e3822deed1f51a677b589224
+size 56754

runs/Feb06_11-48-54_node01/events.out.tfevents.1707195876.node01.4168700.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef1bf451861f1901231ea65683a85d5be9f6232bbdaef7385eb1ec7a5c9fb998
+size 828

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 4096,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.14663515942437308,
+    "train_runtime": 4275.5798,
+    "train_samples": 12879,
+    "train_samples_per_second": 3.012,
+    "train_steps_per_second": 0.188
+}

trainer_state.json

@@ -0,0 +1,1180 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 805,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.172839506172839e-09,
+      "logits/chosen": -2.8421168327331543,
+      "logits/rejected": -2.6747336387634277,
+      "logps/chosen": -92.33953094482422,
+      "logps/rejected": -44.262760162353516,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.172839506172839e-08,
+      "logits/chosen": -2.9818074703216553,
+      "logits/rejected": -2.9741735458374023,
+      "logps/chosen": -197.26390075683594,
+      "logps/rejected": -149.23976135253906,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/chosen": 0.020052114501595497,
+      "rewards/margins": 0.0660661831498146,
+      "rewards/rejected": -0.04601407051086426,
+      "step": 10
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2345679012345677e-07,
+      "logits/chosen": -2.9329466819763184,
+      "logits/rejected": -2.8775200843811035,
+      "logps/chosen": -172.8914337158203,
+      "logps/rejected": -133.22866821289062,
+      "loss": 0.5158,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.48435258865356445,
+      "rewards/margins": 0.6018184423446655,
+      "rewards/rejected": -0.11746580898761749,
+      "step": 20
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8518518518518516e-07,
+      "logits/chosen": -3.0486252307891846,
+      "logits/rejected": -3.003458023071289,
+      "logps/chosen": -178.92950439453125,
+      "logps/rejected": -143.7101287841797,
+      "loss": 0.3374,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.6527904272079468,
+      "rewards/margins": 1.396834135055542,
+      "rewards/rejected": -0.7440436482429504,
+      "step": 30
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4691358024691354e-07,
+      "logits/chosen": -2.970651388168335,
+      "logits/rejected": -2.966358184814453,
+      "logps/chosen": -174.35009765625,
+      "logps/rejected": -139.04750061035156,
+      "loss": 0.2663,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.44792431592941284,
+      "rewards/margins": 2.088245391845703,
+      "rewards/rejected": -1.6403213739395142,
+      "step": 40
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.086419753086419e-07,
+      "logits/chosen": -2.9904656410217285,
+      "logits/rejected": -2.920414686203003,
+      "logps/chosen": -173.70179748535156,
+      "logps/rejected": -138.68955993652344,
+      "loss": 0.1817,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.6215466856956482,
+      "rewards/margins": 2.8885910511016846,
+      "rewards/rejected": -2.2670440673828125,
+      "step": 50
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.703703703703703e-07,
+      "logits/chosen": -3.000448703765869,
+      "logits/rejected": -2.8979151248931885,
+      "logps/chosen": -145.14614868164062,
+      "logps/rejected": -112.61439514160156,
+      "loss": 0.1912,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.05986971780657768,
+      "rewards/margins": 3.269381046295166,
+      "rewards/rejected": -3.2095108032226562,
+      "step": 60
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.320987654320987e-07,
+      "logits/chosen": -2.9225759506225586,
+      "logits/rejected": -2.8518130779266357,
+      "logps/chosen": -157.19439697265625,
+      "logps/rejected": -119.10911560058594,
+      "loss": 0.1713,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": 0.39277735352516174,
+      "rewards/margins": 4.21900749206543,
+      "rewards/rejected": -3.82623028755188,
+      "step": 70
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.938271604938271e-07,
+      "logits/chosen": -2.842589855194092,
+      "logits/rejected": -2.805922031402588,
+      "logps/chosen": -168.32815551757812,
+      "logps/rejected": -141.83216857910156,
+      "loss": 0.1762,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.5003746151924133,
+      "rewards/margins": 5.050185203552246,
+      "rewards/rejected": -5.550559043884277,
+      "step": 80
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.937845303867402e-07,
+      "logits/chosen": -2.9922776222229004,
+      "logits/rejected": -2.970557928085327,
+      "logps/chosen": -172.38787841796875,
+      "logps/rejected": -144.65328979492188,
+      "loss": 0.144,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -2.053427219390869,
+      "rewards/margins": 4.2831034660339355,
+      "rewards/rejected": -6.336531639099121,
+      "step": 90
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.868784530386741e-07,
+      "logits/chosen": -2.897087574005127,
+      "logits/rejected": -2.9255499839782715,
+      "logps/chosen": -183.1656951904297,
+      "logps/rejected": -155.22946166992188,
+      "loss": 0.1584,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 0.06980714946985245,
+      "rewards/margins": 5.842454433441162,
+      "rewards/rejected": -5.772646903991699,
+      "step": 100
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.799723756906077e-07,
+      "logits/chosen": -3.016488790512085,
+      "logits/rejected": -2.9143893718719482,
+      "logps/chosen": -182.10484313964844,
+      "logps/rejected": -151.207275390625,
+      "loss": 0.185,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -1.8662151098251343,
+      "rewards/margins": 4.957070350646973,
+      "rewards/rejected": -6.823284149169922,
+      "step": 110
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.730662983425414e-07,
+      "logits/chosen": -2.906358242034912,
+      "logits/rejected": -2.8886070251464844,
+      "logps/chosen": -162.4820556640625,
+      "logps/rejected": -144.28274536132812,
+      "loss": 0.2087,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.8441675901412964,
+      "rewards/margins": 5.8418402671813965,
+      "rewards/rejected": -6.686008453369141,
+      "step": 120
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.661602209944751e-07,
+      "logits/chosen": -3.044167995452881,
+      "logits/rejected": -2.989940881729126,
+      "logps/chosen": -192.1724090576172,
+      "logps/rejected": -160.28482055664062,
+      "loss": 0.1281,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -1.2753368616104126,
+      "rewards/margins": 5.963367462158203,
+      "rewards/rejected": -7.238703727722168,
+      "step": 130
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.592541436464088e-07,
+      "logits/chosen": -2.9158730506896973,
+      "logits/rejected": -2.8665575981140137,
+      "logps/chosen": -176.7301788330078,
+      "logps/rejected": -157.2458038330078,
+      "loss": 0.121,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 0.29454001784324646,
+      "rewards/margins": 7.677084445953369,
+      "rewards/rejected": -7.38254451751709,
+      "step": 140
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.5234806629834255e-07,
+      "logits/chosen": -2.906966209411621,
+      "logits/rejected": -2.878993034362793,
+      "logps/chosen": -194.67013549804688,
+      "logps/rejected": -177.0902557373047,
+      "loss": 0.1565,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.4568727016448975,
+      "rewards/margins": 5.8625688552856445,
+      "rewards/rejected": -8.319440841674805,
+      "step": 150
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.454419889502762e-07,
+      "logits/chosen": -3.035737991333008,
+      "logits/rejected": -2.9880356788635254,
+      "logps/chosen": -185.5491943359375,
+      "logps/rejected": -158.82350158691406,
+      "loss": 0.1474,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -1.9501594305038452,
+      "rewards/margins": 6.810189247131348,
+      "rewards/rejected": -8.76034927368164,
+      "step": 160
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.3853591160220993e-07,
+      "logits/chosen": -3.051605463027954,
+      "logits/rejected": -2.9627249240875244,
+      "logps/chosen": -188.94488525390625,
+      "logps/rejected": -159.45440673828125,
+      "loss": 0.1317,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -3.886648178100586,
+      "rewards/margins": 6.363480567932129,
+      "rewards/rejected": -10.250128746032715,
+      "step": 170
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.3162983425414365e-07,
+      "logits/chosen": -2.9437613487243652,
+      "logits/rejected": -2.923678398132324,
+      "logps/chosen": -197.8041229248047,
+      "logps/rejected": -179.19410705566406,
+      "loss": 0.1784,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.9531164169311523,
+      "rewards/margins": 5.519814968109131,
+      "rewards/rejected": -8.472932815551758,
+      "step": 180
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.247237569060773e-07,
+      "logits/chosen": -2.9907093048095703,
+      "logits/rejected": -2.91261625289917,
+      "logps/chosen": -183.89828491210938,
+      "logps/rejected": -151.09255981445312,
+      "loss": 0.1128,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.34959203004837036,
+      "rewards/margins": 6.793131351470947,
+      "rewards/rejected": -7.142723083496094,
+      "step": 190
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.1781767955801103e-07,
+      "logits/chosen": -3.0003714561462402,
+      "logits/rejected": -2.9367072582244873,
+      "logps/chosen": -161.82725524902344,
+      "logps/rejected": -141.76551818847656,
+      "loss": 0.1327,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -1.736950159072876,
+      "rewards/margins": 6.6674089431762695,
+      "rewards/rejected": -8.404359817504883,
+      "step": 200
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.1091160220994475e-07,
+      "logits/chosen": -2.9085640907287598,
+      "logits/rejected": -2.866804361343384,
+      "logps/chosen": -186.4488983154297,
+      "logps/rejected": -154.49505615234375,
+      "loss": 0.0995,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.524721622467041,
+      "rewards/margins": 6.8670477867126465,
+      "rewards/rejected": -9.391769409179688,
+      "step": 210
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.0400552486187846e-07,
+      "logits/chosen": -2.9839043617248535,
+      "logits/rejected": -2.905033588409424,
+      "logps/chosen": -196.44332885742188,
+      "logps/rejected": -162.53839111328125,
+      "loss": 0.1511,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -1.4525755643844604,
+      "rewards/margins": 6.9121527671813965,
+      "rewards/rejected": -8.364728927612305,
+      "step": 220
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 3.970994475138121e-07,
+      "logits/chosen": -2.9412007331848145,
+      "logits/rejected": -2.897495746612549,
+      "logps/chosen": -169.09982299804688,
+      "logps/rejected": -140.54649353027344,
+      "loss": 0.1487,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.6759728193283081,
+      "rewards/margins": 5.637655258178711,
+      "rewards/rejected": -6.313628196716309,
+      "step": 230
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 3.9019337016574584e-07,
+      "logits/chosen": -2.9135284423828125,
+      "logits/rejected": -2.9134836196899414,
+      "logps/chosen": -182.82675170898438,
+      "logps/rejected": -172.9473419189453,
+      "loss": 0.1548,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.683964490890503,
+      "rewards/margins": 7.280943393707275,
+      "rewards/rejected": -9.964908599853516,
+      "step": 240
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 3.832872928176795e-07,
+      "logits/chosen": -2.9248108863830566,
+      "logits/rejected": -2.929591655731201,
+      "logps/chosen": -170.30747985839844,
+      "logps/rejected": -158.3248291015625,
+      "loss": 0.0943,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.5063244104385376,
+      "rewards/margins": 8.612382888793945,
+      "rewards/rejected": -9.118706703186035,
+      "step": 250
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 3.763812154696133e-07,
+      "logits/chosen": -2.9988207817077637,
+      "logits/rejected": -2.945227861404419,
+      "logps/chosen": -176.06637573242188,
+      "logps/rejected": -152.9449005126953,
+      "loss": 0.1231,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.20524081587791443,
+      "rewards/margins": 7.8652777671813965,
+      "rewards/rejected": -7.660037994384766,
+      "step": 260
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 3.6947513812154694e-07,
+      "logits/chosen": -3.014315366744995,
+      "logits/rejected": -2.928999185562134,
+      "logps/chosen": -171.6766357421875,
+      "logps/rejected": -147.11154174804688,
+      "loss": 0.1415,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 1.9650657176971436,
+      "rewards/margins": 6.9074506759643555,
+      "rewards/rejected": -4.942384243011475,
+      "step": 270
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 3.6256906077348066e-07,
+      "logits/chosen": -2.907224178314209,
+      "logits/rejected": -2.906038761138916,
+      "logps/chosen": -182.75013732910156,
+      "logps/rejected": -167.35330200195312,
+      "loss": 0.1677,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 0.3685176968574524,
+      "rewards/margins": 7.3049635887146,
+      "rewards/rejected": -6.936445713043213,
+      "step": 280
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 3.556629834254143e-07,
+      "logits/chosen": -2.899120330810547,
+      "logits/rejected": -2.84541654586792,
+      "logps/chosen": -148.26084899902344,
+      "logps/rejected": -133.62619018554688,
+      "loss": 0.1569,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.6295042037963867,
+      "rewards/margins": 6.114689350128174,
+      "rewards/rejected": -5.485184669494629,
+      "step": 290
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.4875690607734804e-07,
+      "logits/chosen": -2.965310573577881,
+      "logits/rejected": -2.8868935108184814,
+      "logps/chosen": -181.91659545898438,
+      "logps/rejected": -153.7183380126953,
+      "loss": 0.1737,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 0.35049858689308167,
+      "rewards/margins": 7.459444999694824,
+      "rewards/rejected": -7.108946323394775,
+      "step": 300
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.418508287292817e-07,
+      "logits/chosen": -2.978998899459839,
+      "logits/rejected": -2.9149255752563477,
+      "logps/chosen": -195.01406860351562,
+      "logps/rejected": -172.47183227539062,
+      "loss": 0.1305,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -8.093232154846191,
+      "rewards/margins": 7.233319282531738,
+      "rewards/rejected": -15.326550483703613,
+      "step": 310
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.3494475138121547e-07,
+      "logits/chosen": -2.976937770843506,
+      "logits/rejected": -2.8859190940856934,
+      "logps/chosen": -199.7006072998047,
+      "logps/rejected": -168.48184204101562,
+      "loss": 0.094,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -6.615779876708984,
+      "rewards/margins": 7.833823204040527,
+      "rewards/rejected": -14.449602127075195,
+      "step": 320
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.280386740331492e-07,
+      "logits/chosen": -2.887190103530884,
+      "logits/rejected": -2.877593517303467,
+      "logps/chosen": -167.93466186523438,
+      "logps/rejected": -152.63137817382812,
+      "loss": 0.1647,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -7.1772050857543945,
+      "rewards/margins": 6.209255218505859,
+      "rewards/rejected": -13.386459350585938,
+      "step": 330
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.2113259668508285e-07,
+      "logits/chosen": -3.028519868850708,
+      "logits/rejected": -2.9872825145721436,
+      "logps/chosen": -209.1421661376953,
+      "logps/rejected": -189.716796875,
+      "loss": 0.1479,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -8.37966251373291,
+      "rewards/margins": 6.489211082458496,
+      "rewards/rejected": -14.868873596191406,
+      "step": 340
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.1422651933701657e-07,
+      "logits/chosen": -2.91877818107605,
+      "logits/rejected": -2.86418080329895,
+      "logps/chosen": -188.1709747314453,
+      "logps/rejected": -169.12794494628906,
+      "loss": 0.12,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -8.10500431060791,
+      "rewards/margins": 7.924190521240234,
+      "rewards/rejected": -16.029193878173828,
+      "step": 350
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.0732044198895023e-07,
+      "logits/chosen": -2.9679653644561768,
+      "logits/rejected": -2.9573211669921875,
+      "logps/chosen": -191.16941833496094,
+      "logps/rejected": -175.7834014892578,
+      "loss": 0.1287,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -6.326230525970459,
+      "rewards/margins": 7.559299468994141,
+      "rewards/rejected": -13.885531425476074,
+      "step": 360
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.00414364640884e-07,
+      "logits/chosen": -2.9169762134552,
+      "logits/rejected": -2.834723949432373,
+      "logps/chosen": -178.90940856933594,
+      "logps/rejected": -150.42605590820312,
+      "loss": 0.0809,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -4.185718059539795,
+      "rewards/margins": 8.697784423828125,
+      "rewards/rejected": -12.883502006530762,
+      "step": 370
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 2.9350828729281767e-07,
+      "logits/chosen": -3.0018362998962402,
+      "logits/rejected": -2.9604392051696777,
+      "logps/chosen": -170.78225708007812,
+      "logps/rejected": -154.3968505859375,
+      "loss": 0.1239,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.6663613319396973,
+      "rewards/margins": 8.932355880737305,
+      "rewards/rejected": -11.598716735839844,
+      "step": 380
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 2.866022099447514e-07,
+      "logits/chosen": -2.994075298309326,
+      "logits/rejected": -2.9975550174713135,
+      "logps/chosen": -215.20736694335938,
+      "logps/rejected": -194.09291076660156,
+      "loss": 0.1248,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -3.196017265319824,
+      "rewards/margins": 10.618438720703125,
+      "rewards/rejected": -13.81445598602295,
+      "step": 390
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.7969613259668505e-07,
+      "logits/chosen": -3.0385982990264893,
+      "logits/rejected": -2.966886043548584,
+      "logps/chosen": -178.92239379882812,
+      "logps/rejected": -152.96282958984375,
+      "loss": 0.1334,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.9255645275115967,
+      "rewards/margins": 9.632636070251465,
+      "rewards/rejected": -11.55820083618164,
+      "step": 400
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.7279005524861877e-07,
+      "logits/chosen": -3.092007637023926,
+      "logits/rejected": -2.971647024154663,
+      "logps/chosen": -212.3997039794922,
+      "logps/rejected": -171.35873413085938,
+      "loss": 0.1333,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.5890955924987793,
+      "rewards/margins": 8.974533081054688,
+      "rewards/rejected": -11.563628196716309,
+      "step": 410
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.6588397790055243e-07,
+      "logits/chosen": -2.9924330711364746,
+      "logits/rejected": -2.9680898189544678,
+      "logps/chosen": -187.8248291015625,
+      "logps/rejected": -179.09518432617188,
+      "loss": 0.1164,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -2.1534762382507324,
+      "rewards/margins": 8.748167991638184,
+      "rewards/rejected": -10.901644706726074,
+      "step": 420
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.589779005524862e-07,
+      "logits/chosen": -3.084371566772461,
+      "logits/rejected": -3.0335347652435303,
+      "logps/chosen": -200.3054962158203,
+      "logps/rejected": -174.8027801513672,
+      "loss": 0.1358,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.3474791646003723,
+      "rewards/margins": 8.830670356750488,
+      "rewards/rejected": -9.17814826965332,
+      "step": 430
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5207182320441986e-07,
+      "logits/chosen": -3.0466129779815674,
+      "logits/rejected": -2.9966206550598145,
+      "logps/chosen": -204.1211395263672,
+      "logps/rejected": -184.7081298828125,
+      "loss": 0.106,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.979719340801239,
+      "rewards/margins": 8.62094783782959,
+      "rewards/rejected": -9.600667953491211,
+      "step": 440
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.451657458563536e-07,
+      "logits/chosen": -3.0778112411499023,
+      "logits/rejected": -3.046935558319092,
+      "logps/chosen": -193.56973266601562,
+      "logps/rejected": -168.67868041992188,
+      "loss": 0.0973,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.372416615486145,
+      "rewards/margins": 8.402765274047852,
+      "rewards/rejected": -9.775181770324707,
+      "step": 450
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3825966850828727e-07,
+      "logits/chosen": -2.967545986175537,
+      "logits/rejected": -2.895456314086914,
+      "logps/chosen": -157.90182495117188,
+      "logps/rejected": -134.09254455566406,
+      "loss": 0.1293,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": 0.49174195528030396,
+      "rewards/margins": 9.59979248046875,
+      "rewards/rejected": -9.108049392700195,
+      "step": 460
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.31353591160221e-07,
+      "logits/chosen": -3.0117809772491455,
+      "logits/rejected": -2.9402668476104736,
+      "logps/chosen": -190.03530883789062,
+      "logps/rejected": -168.36068725585938,
+      "loss": 0.0976,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.2254009246826172,
+      "rewards/margins": 9.530010223388672,
+      "rewards/rejected": -9.755411148071289,
+      "step": 470
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.2444751381215468e-07,
+      "logits/chosen": -3.088365077972412,
+      "logits/rejected": -2.996295928955078,
+      "logps/chosen": -197.9468994140625,
+      "logps/rejected": -172.51962280273438,
+      "loss": 0.0996,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -3.3391284942626953,
+      "rewards/margins": 8.23055648803711,
+      "rewards/rejected": -11.569684982299805,
+      "step": 480
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.175414364640884e-07,
+      "logits/chosen": -2.968456983566284,
+      "logits/rejected": -2.961946964263916,
+      "logps/chosen": -186.1309051513672,
+      "logps/rejected": -169.68812561035156,
+      "loss": 0.1217,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.4508451521396637,
+      "rewards/margins": 11.474364280700684,
+      "rewards/rejected": -11.925207138061523,
+      "step": 490
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 2.1063535911602208e-07,
+      "logits/chosen": -2.978132724761963,
+      "logits/rejected": -2.919520616531372,
+      "logps/chosen": -196.43170166015625,
+      "logps/rejected": -174.66552734375,
+      "loss": 0.1262,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -5.012697696685791,
+      "rewards/margins": 8.436281204223633,
+      "rewards/rejected": -13.448979377746582,
+      "step": 500
+    },
+    {
+      "epoch": 0.62,
+      "eval_logits/chosen": -2.9003899097442627,
+      "eval_logits/rejected": -2.85654616355896,
+      "eval_logps/chosen": -188.17869567871094,
+      "eval_logps/rejected": -171.17919921875,
+      "eval_loss": 0.118459552526474,
+      "eval_rewards/accuracies": 0.8670212626457214,
+      "eval_rewards/chosen": -3.0113587379455566,
+      "eval_rewards/margins": 9.170669555664062,
+      "eval_rewards/rejected": -12.182028770446777,
+      "eval_runtime": 242.1612,
+      "eval_samples_per_second": 6.174,
+      "eval_steps_per_second": 0.194,
+      "step": 500
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 2.0372928176795578e-07,
+      "logits/chosen": -2.944871425628662,
+      "logits/rejected": -2.9157793521881104,
+      "logps/chosen": -180.36338806152344,
+      "logps/rejected": -159.6240692138672,
+      "loss": 0.1055,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -2.646656036376953,
+      "rewards/margins": 9.775663375854492,
+      "rewards/rejected": -12.422318458557129,
+      "step": 510
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.968232044198895e-07,
+      "logits/chosen": -3.010129690170288,
+      "logits/rejected": -2.9667623043060303,
+      "logps/chosen": -171.2137908935547,
+      "logps/rejected": -152.80848693847656,
+      "loss": 0.1331,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -3.5951690673828125,
+      "rewards/margins": 7.591134548187256,
+      "rewards/rejected": -11.186304092407227,
+      "step": 520
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.899171270718232e-07,
+      "logits/chosen": -3.0541915893554688,
+      "logits/rejected": -2.9794061183929443,
+      "logps/chosen": -179.92555236816406,
+      "logps/rejected": -154.8466339111328,
+      "loss": 0.1356,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.0221731662750244,
+      "rewards/margins": 8.44990062713623,
+      "rewards/rejected": -10.472073554992676,
+      "step": 530
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.830110497237569e-07,
+      "logits/chosen": -2.930157423019409,
+      "logits/rejected": -2.9100921154022217,
+      "logps/chosen": -208.7513885498047,
+      "logps/rejected": -196.99765014648438,
+      "loss": 0.1238,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -4.048346519470215,
+      "rewards/margins": 8.839378356933594,
+      "rewards/rejected": -12.887723922729492,
+      "step": 540
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.7610497237569062e-07,
+      "logits/chosen": -2.9872794151306152,
+      "logits/rejected": -2.9665157794952393,
+      "logps/chosen": -181.928955078125,
+      "logps/rejected": -173.3523712158203,
+      "loss": 0.1339,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.8346304893493652,
+      "rewards/margins": 8.800179481506348,
+      "rewards/rejected": -11.634809494018555,
+      "step": 550
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.691988950276243e-07,
+      "logits/chosen": -3.0336689949035645,
+      "logits/rejected": -3.022378444671631,
+      "logps/chosen": -197.01950073242188,
+      "logps/rejected": -182.48306274414062,
+      "loss": 0.1319,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.3179187774658203,
+      "rewards/margins": 9.856887817382812,
+      "rewards/rejected": -12.174806594848633,
+      "step": 560
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.62292817679558e-07,
+      "logits/chosen": -2.9352331161499023,
+      "logits/rejected": -2.8333239555358887,
+      "logps/chosen": -183.40594482421875,
+      "logps/rejected": -157.8167266845703,
+      "loss": 0.0838,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -3.595484495162964,
+      "rewards/margins": 9.1626615524292,
+      "rewards/rejected": -12.758146286010742,
+      "step": 570
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.5538674033149171e-07,
+      "logits/chosen": -3.005654811859131,
+      "logits/rejected": -2.9100375175476074,
+      "logps/chosen": -184.67526245117188,
+      "logps/rejected": -154.762451171875,
+      "loss": 0.1174,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.7798635959625244,
+      "rewards/margins": 10.110002517700195,
+      "rewards/rejected": -11.889866828918457,
+      "step": 580
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.484806629834254e-07,
+      "logits/chosen": -3.069862127304077,
+      "logits/rejected": -2.978449583053589,
+      "logps/chosen": -209.2169647216797,
+      "logps/rejected": -170.54354858398438,
+      "loss": 0.122,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.5865066051483154,
+      "rewards/margins": 8.170557022094727,
+      "rewards/rejected": -10.757063865661621,
+      "step": 590
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 1.4157458563535912e-07,
+      "logits/chosen": -3.0297770500183105,
+      "logits/rejected": -2.956226110458374,
+      "logps/chosen": -179.11427307128906,
+      "logps/rejected": -155.3955535888672,
+      "loss": 0.1675,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.389403820037842,
+      "rewards/margins": 7.711805820465088,
+      "rewards/rejected": -10.10120964050293,
+      "step": 600
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 1.346685082872928e-07,
+      "logits/chosen": -2.928321599960327,
+      "logits/rejected": -2.89027738571167,
+      "logps/chosen": -214.89675903320312,
+      "logps/rejected": -190.57569885253906,
+      "loss": 0.0856,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.7383911609649658,
+      "rewards/margins": 10.676383018493652,
+      "rewards/rejected": -12.414773941040039,
+      "step": 610
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 1.277624309392265e-07,
+      "logits/chosen": -3.0483570098876953,
+      "logits/rejected": -2.9306087493896484,
+      "logps/chosen": -180.202880859375,
+      "logps/rejected": -148.0863037109375,
+      "loss": 0.1345,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.625720977783203,
+      "rewards/margins": 7.224545955657959,
+      "rewards/rejected": -10.85026741027832,
+      "step": 620
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 1.2085635359116022e-07,
+      "logits/chosen": -3.087991237640381,
+      "logits/rejected": -3.0041158199310303,
+      "logps/chosen": -198.82481384277344,
+      "logps/rejected": -169.78836059570312,
+      "loss": 0.1162,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -2.9254348278045654,
+      "rewards/margins": 8.505046844482422,
+      "rewards/rejected": -11.430481910705566,
+      "step": 630
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 1.1395027624309392e-07,
+      "logits/chosen": -2.9629359245300293,
+      "logits/rejected": -2.980262517929077,
+      "logps/chosen": -201.67306518554688,
+      "logps/rejected": -190.01260375976562,
+      "loss": 0.1416,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.385073661804199,
+      "rewards/margins": 8.24610710144043,
+      "rewards/rejected": -12.631181716918945,
+      "step": 640
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 1.0704419889502763e-07,
+      "logits/chosen": -3.059577465057373,
+      "logits/rejected": -2.9308218955993652,
+      "logps/chosen": -221.16049194335938,
+      "logps/rejected": -187.49386596679688,
+      "loss": 0.103,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -3.075377941131592,
+      "rewards/margins": 10.026552200317383,
+      "rewards/rejected": -13.101930618286133,
+      "step": 650
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 1.0013812154696132e-07,
+      "logits/chosen": -3.097217082977295,
+      "logits/rejected": -2.9983904361724854,
+      "logps/chosen": -198.12832641601562,
+      "logps/rejected": -173.28384399414062,
+      "loss": 0.122,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -3.524822950363159,
+      "rewards/margins": 8.267110824584961,
+      "rewards/rejected": -11.791935920715332,
+      "step": 660
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 9.323204419889502e-08,
+      "logits/chosen": -3.0255918502807617,
+      "logits/rejected": -2.9517340660095215,
+      "logps/chosen": -174.54376220703125,
+      "logps/rejected": -154.30923461914062,
+      "loss": 0.1028,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.3350832462310791,
+      "rewards/margins": 10.506619453430176,
+      "rewards/rejected": -10.841703414916992,
+      "step": 670
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 8.632596685082872e-08,
+      "logits/chosen": -2.9522705078125,
+      "logits/rejected": -2.905046224594116,
+      "logps/chosen": -184.12936401367188,
+      "logps/rejected": -156.78311157226562,
+      "loss": 0.1092,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -1.2172771692276,
+      "rewards/margins": 10.139422416687012,
+      "rewards/rejected": -11.35669994354248,
+      "step": 680
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 7.941988950276243e-08,
+      "logits/chosen": -2.9998116493225098,
+      "logits/rejected": -2.93977689743042,
+      "logps/chosen": -159.04739379882812,
+      "logps/rejected": -137.8501434326172,
+      "loss": 0.1154,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.821601390838623,
+      "rewards/margins": 8.011626243591309,
+      "rewards/rejected": -10.833227157592773,
+      "step": 690
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 7.251381215469612e-08,
+      "logits/chosen": -3.064391613006592,
+      "logits/rejected": -3.0105395317077637,
+      "logps/chosen": -216.6563720703125,
+      "logps/rejected": -190.38059997558594,
+      "loss": 0.143,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -3.8847603797912598,
+      "rewards/margins": 9.102531433105469,
+      "rewards/rejected": -12.987292289733887,
+      "step": 700
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 6.560773480662984e-08,
+      "logits/chosen": -2.8895156383514404,
+      "logits/rejected": -2.873819589614868,
+      "logps/chosen": -165.34201049804688,
+      "logps/rejected": -158.01522827148438,
+      "loss": 0.1274,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.2765231132507324,
+      "rewards/margins": 10.176279067993164,
+      "rewards/rejected": -12.452800750732422,
+      "step": 710
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 5.870165745856354e-08,
+      "logits/chosen": -2.992870569229126,
+      "logits/rejected": -2.9288392066955566,
+      "logps/chosen": -166.22451782226562,
+      "logps/rejected": -149.21884155273438,
+      "loss": 0.0849,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.1316981315612793,
+      "rewards/margins": 11.313945770263672,
+      "rewards/rejected": -11.44564437866211,
+      "step": 720
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 5.1795580110497236e-08,
+      "logits/chosen": -3.0519726276397705,
+      "logits/rejected": -2.9871127605438232,
+      "logps/chosen": -198.3760528564453,
+      "logps/rejected": -173.3822021484375,
+      "loss": 0.1439,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -4.240297317504883,
+      "rewards/margins": 9.328596115112305,
+      "rewards/rejected": -13.568893432617188,
+      "step": 730
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 4.488950276243094e-08,
+      "logits/chosen": -2.905094623565674,
+      "logits/rejected": -2.898705005645752,
+      "logps/chosen": -166.13040161132812,
+      "logps/rejected": -166.46963500976562,
+      "loss": 0.0909,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.6403870582580566,
+      "rewards/margins": 10.037248611450195,
+      "rewards/rejected": -12.677635192871094,
+      "step": 740
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 3.7983425414364637e-08,
+      "logits/chosen": -3.0315964221954346,
+      "logits/rejected": -2.945363998413086,
+      "logps/chosen": -186.29241943359375,
+      "logps/rejected": -159.93798828125,
+      "loss": 0.1194,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -3.548847198486328,
+      "rewards/margins": 9.53381061553955,
+      "rewards/rejected": -13.082656860351562,
+      "step": 750
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 3.107734806629834e-08,
+      "logits/chosen": -2.901912212371826,
+      "logits/rejected": -2.8841569423675537,
+      "logps/chosen": -165.88009643554688,
+      "logps/rejected": -153.7960662841797,
+      "loss": 0.1276,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -3.673292875289917,
+      "rewards/margins": 10.183616638183594,
+      "rewards/rejected": -13.856908798217773,
+      "step": 760
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.4171270718232044e-08,
+      "logits/chosen": -3.0154058933258057,
+      "logits/rejected": -2.9658362865448,
+      "logps/chosen": -207.61538696289062,
+      "logps/rejected": -184.4959259033203,
+      "loss": 0.1581,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -3.740403652191162,
+      "rewards/margins": 10.590685844421387,
+      "rewards/rejected": -14.331089973449707,
+      "step": 770
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7265193370165747e-08,
+      "logits/chosen": -3.099858045578003,
+      "logits/rejected": -2.996018171310425,
+      "logps/chosen": -209.8662109375,
+      "logps/rejected": -179.3186492919922,
+      "loss": 0.0908,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.9476644396781921,
+      "rewards/margins": 12.100104331970215,
+      "rewards/rejected": -13.047770500183105,
+      "step": 780
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 1.0359116022099446e-08,
+      "logits/chosen": -3.0576415061950684,
+      "logits/rejected": -2.9471096992492676,
+      "logps/chosen": -175.6331787109375,
+      "logps/rejected": -154.61416625976562,
+      "loss": 0.0915,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -3.3139986991882324,
+      "rewards/margins": 9.281652450561523,
+      "rewards/rejected": -12.595651626586914,
+      "step": 790
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.453038674033149e-09,
+      "logits/chosen": -2.9590699672698975,
+      "logits/rejected": -2.9163191318511963,
+      "logps/chosen": -206.9409942626953,
+      "logps/rejected": -183.0679931640625,
+      "loss": 0.1379,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -2.519080638885498,
+      "rewards/margins": 11.627119064331055,
+      "rewards/rejected": -14.146200180053711,
+      "step": 800
+    },
+    {
+      "epoch": 1.0,
+      "step": 805,
+      "total_flos": 0.0,
+      "train_loss": 0.14663515942437308,
+      "train_runtime": 4275.5798,
+      "train_samples_per_second": 3.012,
+      "train_steps_per_second": 0.188
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 805,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b94bb82d8150b3611e68e1c86ea43db247708d7d74f6edcd32b197dac16943c9
+size 5371