Model save

README.md

@@ -1,5 +1,5 @@
 ---
-base_model: Qwen/Qwen2.5-Math-7B
+base_model: Qwen/Qwen2.5-0.5B
 library_name: transformers
 model_name: Qwen-2.5-7B-Simple-RL
 tags:
@@ -11,7 +11,7 @@ licence: license
 
 # Model Card for Qwen-2.5-7B-Simple-RL
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-Math-7B](https://huggingface.co/Qwen/Qwen2.5-Math-7B).
+This model is a fine-tuned version of [Qwen/Qwen2.5-0.5B](https://huggingface.co/Qwen/Qwen2.5-0.5B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "Qwen/Qwen2.5-Math-7B",
+  "base_model_name_or_path": "Qwen/Qwen2.5-0.5B",
   "bias": "none",
   "eva_config": null,
   "exclude_modules": null,
@@ -23,13 +23,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "o_proj",
-    "v_proj",
-    "gate_proj",
-    "k_proj",
     "q_proj",
-    "down_proj"
+    "k_proj",
+    "gate_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8d1299b48026bfa4882d567b9562ebfc2772388a281c3593858b688dd0446f7c
-size 161533192
+oid sha256:287246a45303c5a200b655409bf837910a83e9fb80d67a180643de6dcdefbcfc
+size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0008721697144210338,
-    "train_runtime": 2002.6709,
+    "train_loss": 0.0028079951708861748,
+    "train_runtime": 39617.8147,
     "train_samples": 7500,
-    "train_samples_per_second": 0.24,
-    "train_steps_per_second": 0.015
+    "train_samples_per_second": 0.189,
+    "train_steps_per_second": 0.012
 }

config.json

@@ -1,5 +1,6 @@
 {
-  "_name_or_path": "Qwen/Qwen2.5-0.5B",
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "Qwen/Qwen2.5-Math-7B",
   "architectures": [
     "Qwen2ForCausalLM"
   ],
@@ -7,24 +8,24 @@
   "bos_token_id": 151643,
   "eos_token_id": 151643,
   "hidden_act": "silu",
-  "hidden_size": 896,
+  "hidden_size": 3584,
   "initializer_range": 0.02,
-  "intermediate_size": 4864,
-  "max_position_embeddings": 32768,
-  "max_window_layers": 24,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 4096,
+  "max_window_layers": 28,
   "model_type": "qwen2",
-  "num_attention_heads": 14,
-  "num_hidden_layers": 24,
-  "num_key_value_heads": 2,
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
   "rms_norm_eps": 1e-06,
   "rope_scaling": null,
-  "rope_theta": 1000000.0,
-  "sliding_window": 32768,
-  "tie_word_embeddings": true,
+  "rope_theta": 10000,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
   "transformers_version": "4.49.0",
   "use_cache": true,
   "use_mrope": false,
   "use_sliding_window": false,
-  "vocab_size": 151936
+  "vocab_size": 152064
 }

tokenizer_config.json

@@ -195,7 +195,7 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'Please reason step by step, and put your final answer within \\\\boxed{}.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nPlease reason step by step, and put your final answer within \\\\boxed{}.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "errors": "replace",

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0008721697144210338,
-    "train_runtime": 2002.6709,
+    "train_loss": 0.0028079951708861748,
+    "train_runtime": 39617.8147,
     "train_samples": 7500,
-    "train_samples_per_second": 0.24,
-    "train_steps_per_second": 0.015
+    "train_samples_per_second": 0.189,
+    "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -1,20 +1,20 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.032,
+  "epoch": 0.9984,
   "eval_steps": 100,
-  "global_step": 30,
+  "global_step": 468,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 246.8125,
-      "epoch": 0.0010666666666666667,
+      "completion_length": 599.40625,
+      "epoch": 0.0021333333333333334,
       "grad_norm": 0.0,
       "kl": 0.0,
-      "learning_rate": 1e-06,
+      "learning_rate": 6.382978723404255e-08,
       "loss": 0.0,
       "reward": 0.0,
       "reward_std": 0.0,
@@ -24,100 +24,1357 @@
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 228.984375,
-      "epoch": 0.005333333333333333,
-      "grad_norm": 0.0002897010708693415,
-      "kl": 0.0002434550688121817,
-      "learning_rate": 2.959567305869736e-06,
-      "loss": -0.0114,
+      "completion_length": 567.0078125,
+      "epoch": 0.010666666666666666,
+      "grad_norm": 0.02547222189605236,
+      "kl": 0.00040041709871729836,
+      "learning_rate": 3.1914893617021275e-07,
+      "loss": -0.0109,
+      "reward": 0.0859375,
+      "reward_std": 0.07733980286866426,
+      "rewards/accuracy_reward": 0.0859375,
+      "rewards/format_reward": 0.0,
+      "step": 5
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 504.56875,
+      "epoch": 0.021333333333333333,
+      "grad_norm": 0.016505707055330276,
+      "kl": 0.0006629314979363698,
+      "learning_rate": 6.382978723404255e-07,
+      "loss": -0.0096,
+      "reward": 0.04375,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 10
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 505.84375,
+      "epoch": 0.032,
+      "grad_norm": 0.033047277480363846,
+      "kl": 0.0005872767855180427,
+      "learning_rate": 9.574468085106384e-07,
+      "loss": 0.0176,
+      "reward": 0.0375,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 15
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 543.25625,
+      "epoch": 0.042666666666666665,
+      "grad_norm": 0.03408794477581978,
+      "kl": 0.0006242210474738386,
+      "learning_rate": 1.276595744680851e-06,
+      "loss": 0.0039,
+      "reward": 0.04375,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 20
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 597.21875,
+      "epoch": 0.05333333333333334,
+      "grad_norm": 0.02023429423570633,
+      "kl": 0.000612794059998123,
+      "learning_rate": 1.5957446808510639e-06,
+      "loss": 0.0064,
+      "reward": 0.05,
+      "reward_std": 0.07071067690849304,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 25
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 591.35625,
+      "epoch": 0.064,
+      "grad_norm": 0.027683651074767113,
+      "kl": 0.0005714267717848998,
+      "learning_rate": 1.9148936170212767e-06,
+      "loss": -0.0076,
+      "reward": 0.05,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 516.53125,
+      "epoch": 0.07466666666666667,
+      "grad_norm": 0.0005982788279652596,
+      "kl": 0.0005893220724829007,
+      "learning_rate": 2.2340425531914894e-06,
+      "loss": 0.0301,
+      "reward": 0.08125,
+      "reward_std": 0.09722718074917794,
+      "rewards/accuracy_reward": 0.08125,
+      "rewards/format_reward": 0.0,
+      "step": 35
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 544.31875,
+      "epoch": 0.08533333333333333,
+      "grad_norm": 0.0005049636238254607,
+      "kl": 0.0006337692822853569,
+      "learning_rate": 2.553191489361702e-06,
+      "loss": 0.0217,
+      "reward": 0.08125,
+      "reward_std": 0.09722718074917794,
+      "rewards/accuracy_reward": 0.08125,
+      "rewards/format_reward": 0.0,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 514.3875,
+      "epoch": 0.096,
+      "grad_norm": 0.03791235387325287,
+      "kl": 0.0006594170117750764,
+      "learning_rate": 2.872340425531915e-06,
+      "loss": -0.0038,
       "reward": 0.0625,
-      "reward_std": 0.0883883461356163,
+      "reward_std": 0.07071067690849304,
       "rewards/accuracy_reward": 0.0625,
       "rewards/format_reward": 0.0,
-      "step": 5
+      "step": 45
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 234.275,
-      "epoch": 0.010666666666666666,
-      "grad_norm": 0.012982388027012348,
-      "kl": 0.00034311858398723417,
-      "learning_rate": 2.529362456803101e-06,
-      "loss": -0.0046,
+      "completion_length": 509.44375,
+      "epoch": 0.10666666666666667,
+      "grad_norm": 0.02021694928407669,
+      "kl": 0.0006204415323736612,
+      "learning_rate": 2.9996241442585123e-06,
+      "loss": 0.0034,
+      "reward": 0.0375,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 615.1375,
+      "epoch": 0.11733333333333333,
+      "grad_norm": 0.017503464594483376,
+      "kl": 0.0005966832359263208,
+      "learning_rate": 2.9973279301399446e-06,
+      "loss": 0.0078,
+      "reward": 0.04375,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 55
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 508.95625,
+      "epoch": 0.128,
+      "grad_norm": 0.04325263574719429,
+      "kl": 0.0007028729422017932,
+      "learning_rate": 2.992947502998804e-06,
+      "loss": -0.0118,
       "reward": 0.0625,
-      "reward_std": 0.01767766922712326,
+      "reward_std": 0.05303300768136978,
       "rewards/accuracy_reward": 0.0625,
       "rewards/format_reward": 0.0,
-      "step": 10
+      "step": 60
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 239.0,
-      "epoch": 0.016,
-      "grad_norm": 0.00560641847550869,
-      "kl": 0.000386229114519665,
-      "learning_rate": 1.7604722665003958e-06,
-      "loss": 0.0031,
-      "reward": 0.1375,
-      "reward_std": 0.12374368458986282,
-      "rewards/accuracy_reward": 0.1375,
+      "completion_length": 528.80625,
+      "epoch": 0.13866666666666666,
+      "grad_norm": 0.026006193831562996,
+      "kl": 0.0006322090062894859,
+      "learning_rate": 2.9864889601923268e-06,
+      "loss": -0.0065,
+      "reward": 0.06875,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.06875,
       "rewards/format_reward": 0.0,
-      "step": 15
+      "step": 65
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 244.1125,
-      "epoch": 0.021333333333333333,
-      "grad_norm": 0.0002242429181933403,
-      "kl": 0.00041347902733832596,
-      "learning_rate": 9.058803509412648e-07,
-      "loss": 0.0022,
+      "completion_length": 508.56875,
+      "epoch": 0.14933333333333335,
+      "grad_norm": 0.03273686766624451,
+      "kl": 0.0005886858463782119,
+      "learning_rate": 2.977961291721137e-06,
+      "loss": 0.0017,
+      "reward": 0.06875,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.0,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 564.89375,
+      "epoch": 0.16,
+      "grad_norm": 0.012765306048095226,
+      "kl": 0.0005601999211648945,
+      "learning_rate": 2.9673763677155655e-06,
+      "loss": -0.0157,
       "reward": 0.05,
       "reward_std": 0.03535533845424652,
       "rewards/accuracy_reward": 0.05,
       "rewards/format_reward": 0.0,
-      "step": 20
+      "step": 75
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 560.225,
+      "epoch": 0.17066666666666666,
+      "grad_norm": 0.023025022819638252,
+      "kl": 0.0007396811590297148,
+      "learning_rate": 2.9547489219129666e-06,
+      "loss": 0.005,
+      "reward": 0.04375,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 534.0,
+      "epoch": 0.18133333333333335,
+      "grad_norm": 0.0005447011208161712,
+      "kl": 0.0005739938373153563,
+      "learning_rate": 2.9400965311490175e-06,
+      "loss": 0.0065,
+      "reward": 0.025,
+      "reward_std": 0.03535533845424652,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.0,
+      "step": 85
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 578.45625,
+      "epoch": 0.192,
+      "grad_norm": 0.05320625752210617,
+      "kl": 0.0005843402541358955,
+      "learning_rate": 2.9234395908915565e-06,
+      "loss": 0.0148,
+      "reward": 0.05625,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 529.00625,
+      "epoch": 0.20266666666666666,
+      "grad_norm": 0.0005686267395503819,
+      "kl": 0.0006063115164579358,
+      "learning_rate": 2.904801286851009e-06,
+      "loss": -0.0111,
+      "reward": 0.025,
+      "reward_std": 0.03535533845424652,
+      "rewards/accuracy_reward": 0.025,
+      "rewards/format_reward": 0.0,
+      "step": 95
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "grad_norm": 0.0004754478286486119,
+      "learning_rate": 2.884207562706925e-06,
+      "loss": -0.0045,
+      "step": 100
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "eval_clip_ratio": 0.0,
+      "eval_completion_length": 546.1873,
+      "eval_kl": 0.0006224736573698465,
+      "eval_loss": 0.0016505873063579202,
+      "eval_reward": 0.0431,
+      "eval_reward_std": 0.05246732226610184,
+      "eval_rewards/accuracy_reward": 0.0431,
+      "eval_rewards/format_reward": 0.0,
+      "eval_runtime": 6984.4091,
+      "eval_samples_per_second": 0.716,
+      "eval_steps_per_second": 0.179,
+      "step": 100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 504.8,
+      "epoch": 0.224,
+      "grad_norm": 0.043887630105018616,
+      "kl": 0.0005905847807298415,
+      "learning_rate": 2.8616870839955444e-06,
+      "loss": -0.0069,
+      "reward": 0.065625,
+      "reward_std": 0.08396892882883548,
+      "rewards/accuracy_reward": 0.065625,
+      "rewards/format_reward": 0.0,
+      "step": 105
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 554.9875,
+      "epoch": 0.23466666666666666,
+      "grad_norm": 0.03810940682888031,
+      "kl": 0.0006392004950612317,
+      "learning_rate": 2.837271198208662e-06,
+      "loss": 0.0054,
+      "reward": 0.05625,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 594.39375,
+      "epoch": 0.24533333333333332,
+      "grad_norm": 0.0005416510975919664,
+      "kl": 0.0006476349917647894,
+      "learning_rate": 2.8109938911593322e-06,
+      "loss": -0.0032,
+      "reward": 0.00625,
+      "reward_std": 0.00883883461356163,
+      "rewards/accuracy_reward": 0.00625,
+      "rewards/format_reward": 0.0,
+      "step": 115
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 535.65625,
+      "epoch": 0.256,
+      "grad_norm": 0.021277422085404396,
+      "kl": 0.000646474702807609,
+      "learning_rate": 2.7828917396751474e-06,
+      "loss": 0.0019,
+      "reward": 0.05,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 120
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 497.81875,
+      "epoch": 0.26666666666666666,
+      "grad_norm": 0.04059675708413124,
+      "kl": 0.0007086705743859056,
+      "learning_rate": 2.753003860684943e-06,
+      "loss": 0.0073,
+      "reward": 0.03125,
+      "reward_std": 0.02651650384068489,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.0,
+      "step": 125
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 539.66875,
+      "epoch": 0.2773333333333333,
+      "grad_norm": 0.0311752837151289,
+      "kl": 0.0006064358283765614,
+      "learning_rate": 2.721371856769793e-06,
+      "loss": 0.0056,
+      "reward": 0.04375,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 130
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 461.6625,
+      "epoch": 0.288,
+      "grad_norm": 0.018207967281341553,
+      "kl": 0.0006391550123225898,
+      "learning_rate": 2.688039758254093e-06,
+      "loss": 0.0073,
+      "reward": 0.06875,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.0,
+      "step": 135
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 555.3,
+      "epoch": 0.2986666666666667,
+      "grad_norm": 0.014577426947653294,
+      "kl": 0.0006190024098032155,
+      "learning_rate": 2.65305396191733e-06,
+      "loss": 0.0091,
+      "reward": 0.03125,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.0,
+      "step": 140
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 529.79375,
+      "epoch": 0.30933333333333335,
+      "grad_norm": 0.0005597746348939836,
+      "kl": 0.0007494706653233152,
+      "learning_rate": 2.61646316641186e-06,
+      "loss": 0.0059,
+      "reward": 0.05625,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 145
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 533.4125,
+      "epoch": 0.32,
+      "grad_norm": 0.0008026298601180315,
+      "kl": 0.000564340456185164,
+      "learning_rate": 2.5783183044765715e-06,
+      "loss": 0.0155,
+      "reward": 0.04375,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 150
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 572.88125,
+      "epoch": 0.33066666666666666,
+      "grad_norm": 0.02461259439587593,
+      "kl": 0.0006232189080037642,
+      "learning_rate": 2.5386724720408135e-06,
+      "loss": 0.0042,
+      "reward": 0.0625,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.0,
+      "step": 155
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 469.73125,
+      "epoch": 0.3413333333333333,
+      "grad_norm": 0.0007159389788284898,
+      "kl": 0.0006267319105973002,
+      "learning_rate": 2.49758085431725e-06,
+      "loss": -0.0023,
+      "reward": 0.0625,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.0,
+      "step": 160
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 537.625,
+      "epoch": 0.352,
+      "grad_norm": 0.0005227501387707889,
+      "kl": 0.0006310752982244594,
+      "learning_rate": 2.455100648986533e-06,
+      "loss": 0.0019,
+      "reward": 0.05,
+      "reward_std": 0.07071067690849304,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 165
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 570.20625,
+      "epoch": 0.3626666666666667,
+      "grad_norm": 0.0317506417632103,
+      "kl": 0.0006065379689971451,
+      "learning_rate": 2.4112909865807053e-06,
+      "loss": 0.0383,
+      "reward": 0.08125,
+      "reward_std": 0.1149048499763012,
+      "rewards/accuracy_reward": 0.08125,
+      "rewards/format_reward": 0.0,
+      "step": 170
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 548.31875,
+      "epoch": 0.37333333333333335,
+      "grad_norm": 0.04262116923928261,
+      "kl": 0.0005954368047241587,
+      "learning_rate": 2.366212848176164e-06,
+      "loss": -0.0139,
+      "reward": 0.08125,
+      "reward_std": 0.09722718074917794,
+      "rewards/accuracy_reward": 0.08125,
+      "rewards/format_reward": 0.0,
+      "step": 175
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 506.475,
+      "epoch": 0.384,
+      "grad_norm": 0.029712267220020294,
+      "kl": 0.0006351386713504325,
+      "learning_rate": 2.319928980510752e-06,
+      "loss": -0.0088,
+      "reward": 0.06875,
+      "reward_std": 0.09722718074917794,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.0,
+      "step": 180
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 553.3,
+      "epoch": 0.39466666666666667,
+      "grad_norm": 0.01826683059334755,
+      "kl": 0.000638010091643082,
+      "learning_rate": 2.272503808643123e-06,
+      "loss": -0.0008,
+      "reward": 0.04375,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 185
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 572.2125,
+      "epoch": 0.4053333333333333,
+      "grad_norm": 0.025073450058698654,
+      "kl": 0.0006133078852144536,
+      "learning_rate": 2.2240033462759628e-06,
+      "loss": 0.0062,
+      "reward": 0.0625,
+      "reward_std": 0.07071067690849304,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.0,
+      "step": 190
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 529.24375,
+      "epoch": 0.416,
+      "grad_norm": 0.017348134890198708,
+      "kl": 0.0006518879818031564,
+      "learning_rate": 2.1744951038678905e-06,
+      "loss": 0.0129,
+      "reward": 0.0875,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0875,
+      "rewards/format_reward": 0.0,
+      "step": 195
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "grad_norm": 0.0005302979261614382,
+      "learning_rate": 2.124047994661941e-06,
+      "loss": -0.0062,
+      "step": 200
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "eval_clip_ratio": 0.0,
+      "eval_completion_length": 542.9258,
+      "eval_kl": 0.0006711028324672952,
+      "eval_loss": 0.0005841926322318614,
+      "eval_reward": 0.0475,
+      "eval_reward_std": 0.05670996288061142,
+      "eval_rewards/accuracy_reward": 0.0475,
+      "eval_rewards/format_reward": 0.0,
+      "eval_runtime": 6903.4955,
+      "eval_samples_per_second": 0.724,
+      "eval_steps_per_second": 0.181,
+      "step": 200
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 571.925,
+      "epoch": 0.43733333333333335,
+      "grad_norm": 0.03621024638414383,
+      "kl": 0.000652521890879143,
+      "learning_rate": 2.072732238761434e-06,
+      "loss": 0.0023,
+      "reward": 0.04375,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 205
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 556.3875,
+      "epoch": 0.448,
+      "grad_norm": 0.01959877274930477,
+      "kl": 0.0006465273501817137,
+      "learning_rate": 2.0206192653867536e-06,
+      "loss": -0.0093,
+      "reward": 0.04375,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 210
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 524.30625,
+      "epoch": 0.45866666666666667,
+      "grad_norm": 0.026872603222727776,
+      "kl": 0.0006922339722223114,
+      "learning_rate": 1.967781613449095e-06,
+      "loss": 0.0152,
+      "reward": 0.0625,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0625,
+      "rewards/format_reward": 0.0,
+      "step": 215
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 524.2,
+      "epoch": 0.4693333333333333,
+      "grad_norm": 0.024317504838109016,
+      "kl": 0.0007104755510226824,
+      "learning_rate": 1.9142928305795637e-06,
+      "loss": 0.005,
+      "reward": 0.0375,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 220
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 566.8,
+      "epoch": 0.48,
+      "grad_norm": 0.0005844333791173995,
+      "kl": 0.0007887360618042294,
+      "learning_rate": 1.8602273707541886e-06,
+      "loss": -0.004,
+      "reward": 0.01875,
+      "reward_std": 0.02651650384068489,
+      "rewards/accuracy_reward": 0.01875,
+      "rewards/format_reward": 0.0,
+      "step": 225
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 566.1625,
+      "epoch": 0.49066666666666664,
+      "grad_norm": 0.042061224579811096,
+      "kl": 0.0006022223657055292,
+      "learning_rate": 1.8056604906573418e-06,
+      "loss": 0.0171,
+      "reward": 0.05625,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 230
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 589.93125,
+      "epoch": 0.5013333333333333,
+      "grad_norm": 0.017109017819166183,
+      "kl": 0.0006615983191295526,
+      "learning_rate": 1.7506681449278226e-06,
+      "loss": 0.0003,
+      "reward": 0.05625,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 235
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 582.16875,
+      "epoch": 0.512,
+      "grad_norm": 0.02843085303902626,
+      "kl": 0.0006830113568867091,
+      "learning_rate": 1.6953268804334257e-06,
+      "loss": 0.008,
+      "reward": 0.03125,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.0,
+      "step": 240
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 550.20625,
+      "epoch": 0.5226666666666666,
+      "grad_norm": 0.0007241690182127059,
+      "kl": 0.0007094932108884678,
+      "learning_rate": 1.6397137297211436e-06,
+      "loss": 0.0047,
+      "reward": 0.03125,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.0,
+      "step": 245
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 242.75,
-      "epoch": 0.02666666666666667,
-      "grad_norm": 0.016108760610222816,
-      "kl": 0.0003348791698954301,
-      "learning_rate": 2.467682828805956e-07,
+      "completion_length": 520.5375,
+      "epoch": 0.5333333333333333,
+      "grad_norm": 0.013676224276423454,
+      "kl": 0.0006993950722971931,
+      "learning_rate": 1.5839061037913395e-06,
+      "loss": -0.0035,
+      "reward": 0.0375,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 250
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 570.4625,
+      "epoch": 0.544,
+      "grad_norm": 0.026784850284457207,
+      "kl": 0.0007027815772744361,
+      "learning_rate": 1.527981684345115e-06,
+      "loss": 0.0115,
+      "reward": 0.05625,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 255
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 526.28125,
+      "epoch": 0.5546666666666666,
+      "grad_norm": 0.027511239051818848,
+      "kl": 0.0007118464120139834,
+      "learning_rate": 1.4720183156548855e-06,
       "loss": 0.0156,
-      "reward": 0.1375,
-      "reward_std": 0.0883883461356163,
-      "rewards/accuracy_reward": 0.1375,
+      "reward": 0.05625,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.05625,
       "rewards/format_reward": 0.0,
-      "step": 25
+      "step": 260
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 234.625,
-      "epoch": 0.032,
-      "grad_norm": 0.0001786644133972004,
-      "kl": 0.0004210583254462108,
-      "learning_rate": 0.0,
+      "completion_length": 501.05,
+      "epoch": 0.5653333333333334,
+      "grad_norm": 0.0400506816804409,
+      "kl": 0.0007247071487654466,
+      "learning_rate": 1.4160938962086612e-06,
+      "loss": 0.0056,
+      "reward": 0.05,
+      "reward_std": 0.07071067690849304,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 265
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 492.38125,
+      "epoch": 0.576,
+      "grad_norm": 0.020546777173876762,
+      "kl": 0.0006569563163793645,
+      "learning_rate": 1.3602862702788567e-06,
+      "loss": -0.0087,
+      "reward": 0.06875,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.0,
+      "step": 270
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 601.91875,
+      "epoch": 0.5866666666666667,
+      "grad_norm": 0.0006509393570013344,
+      "kl": 0.000718773623520974,
+      "learning_rate": 1.3046731195665748e-06,
       "loss": -0.0019,
-      "reward": 0.0625,
+      "reward": 0.03125,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.0,
+      "step": 275
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 544.3875,
+      "epoch": 0.5973333333333334,
+      "grad_norm": 0.015962699428200722,
+      "kl": 0.0007544517939095385,
+      "learning_rate": 1.2493318550721775e-06,
+      "loss": 0.0026,
+      "reward": 0.0375,
+      "reward_std": 0.03535533845424652,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 280
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 513.525,
+      "epoch": 0.608,
+      "grad_norm": 0.02608601376414299,
+      "kl": 0.0006703489023493602,
+      "learning_rate": 1.1943395093426585e-06,
+      "loss": -0.0063,
+      "reward": 0.05625,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 285
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 537.90625,
+      "epoch": 0.6186666666666667,
+      "grad_norm": 0.0007309815846383572,
+      "kl": 0.0008104784224997275,
+      "learning_rate": 1.1397726292458115e-06,
+      "loss": 0.0138,
+      "reward": 0.04375,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 290
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 530.6,
+      "epoch": 0.6293333333333333,
+      "grad_norm": 0.034363504499197006,
+      "kl": 0.0007202902772405651,
+      "learning_rate": 1.085707169420437e-06,
+      "loss": 0.0225,
+      "reward": 0.04375,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 295
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 0.03348564729094505,
+      "learning_rate": 1.0322183865509054e-06,
+      "loss": 0.0041,
+      "step": 300
+    },
+    {
+      "epoch": 0.64,
+      "eval_clip_ratio": 0.0,
+      "eval_completion_length": 543.87,
+      "eval_kl": 0.0007330777346272952,
+      "eval_loss": 0.002262996742501855,
+      "eval_reward": 0.0493,
+      "eval_reward_std": 0.05840701912641525,
+      "eval_rewards/accuracy_reward": 0.0493,
+      "eval_rewards/format_reward": 0.0,
+      "eval_runtime": 6924.8337,
+      "eval_samples_per_second": 0.722,
+      "eval_steps_per_second": 0.181,
+      "step": 300
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 553.821875,
+      "epoch": 0.6506666666666666,
+      "grad_norm": 0.05884533375501633,
+      "kl": 0.0007077409793055268,
+      "learning_rate": 9.793807346132464e-07,
+      "loss": 0.0064,
+      "reward": 0.065625,
+      "reward_std": 0.09280776344239712,
+      "rewards/accuracy_reward": 0.065625,
+      "rewards/format_reward": 0.0,
+      "step": 305
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 507.25625,
+      "epoch": 0.6613333333333333,
+      "grad_norm": 0.00128034723456949,
+      "kl": 0.0007662738556973636,
+      "learning_rate": 9.272677612385667e-07,
+      "loss": -0.0022,
+      "reward": 0.05625,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 310
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 585.83125,
+      "epoch": 0.672,
+      "grad_norm": 0.027511116117239,
+      "kl": 0.0007258923047629651,
+      "learning_rate": 8.759520053380591e-07,
+      "loss": 0.0104,
+      "reward": 0.0375,
       "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 315
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 566.825,
+      "epoch": 0.6826666666666666,
+      "grad_norm": 0.02675638347864151,
+      "kl": 0.0008431313741311897,
+      "learning_rate": 8.255048961321088e-07,
+      "loss": 0.0081,
+      "reward": 0.0375,
+      "reward_std": 0.03535533845424652,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 320
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 561.75,
+      "epoch": 0.6933333333333334,
+      "grad_norm": 0.03926326334476471,
+      "kl": 0.0007520209066569806,
+      "learning_rate": 7.759966537240373e-07,
+      "loss": -0.0255,
+      "reward": 0.09375,
+      "reward_std": 0.09722718074917794,
+      "rewards/accuracy_reward": 0.09375,
+      "rewards/format_reward": 0.0,
+      "step": 325
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 602.11875,
+      "epoch": 0.704,
+      "grad_norm": 0.04235618934035301,
+      "kl": 0.0006708401182550005,
+      "learning_rate": 7.274961913568773e-07,
+      "loss": -0.0017,
+      "reward": 0.0625,
+      "reward_std": 0.0883883461356163,
       "rewards/accuracy_reward": 0.0625,
       "rewards/format_reward": 0.0,
-      "step": 30
+      "step": 330
     },
     {
-      "epoch": 0.032,
-      "step": 30,
+      "clip_ratio": 0.0,
+      "completion_length": 539.75625,
+      "epoch": 0.7146666666666667,
+      "grad_norm": 0.01949181966483593,
+      "kl": 0.0007214361037767957,
+      "learning_rate": 6.800710194892484e-07,
+      "loss": -0.0142,
+      "reward": 0.05,
+      "reward_std": 0.07071067690849304,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 335
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 559.64375,
+      "epoch": 0.7253333333333334,
+      "grad_norm": 0.01706821843981743,
+      "kl": 0.0007962717980262823,
+      "learning_rate": 6.33787151823836e-07,
+      "loss": 0.0083,
+      "reward": 0.05,
+      "reward_std": 0.03535533845424652,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 340
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 543.775,
+      "epoch": 0.736,
+      "grad_norm": 0.026210952550172806,
+      "kl": 0.000737834313622443,
+      "learning_rate": 5.887090134192947e-07,
+      "loss": 0.0213,
+      "reward": 0.05625,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 345
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 512.84375,
+      "epoch": 0.7466666666666667,
+      "grad_norm": 0.026430701836943626,
+      "kl": 0.0007904948972282,
+      "learning_rate": 5.448993510134669e-07,
+      "loss": -0.0074,
+      "reward": 0.03125,
+      "reward_std": 0.02651650384068489,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.0,
+      "step": 350
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 498.09375,
+      "epoch": 0.7573333333333333,
+      "grad_norm": 0.039444226771593094,
+      "kl": 0.0007300119003048166,
+      "learning_rate": 5.024191456827498e-07,
+      "loss": 0.0032,
+      "reward": 0.05,
+      "reward_std": 0.07071067690849304,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 355
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 534.31875,
+      "epoch": 0.768,
+      "grad_norm": 0.038000140339136124,
+      "kl": 0.000727705340977991,
+      "learning_rate": 4.6132752795918667e-07,
+      "loss": -0.0045,
+      "reward": 0.05,
+      "reward_std": 0.07071067690849304,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 360
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 554.78125,
+      "epoch": 0.7786666666666666,
+      "grad_norm": 0.04807225242257118,
+      "kl": 0.0009294274394051171,
+      "learning_rate": 4.2168169552342905e-07,
+      "loss": -0.0011,
+      "reward": 0.05,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 365
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 552.01875,
+      "epoch": 0.7893333333333333,
+      "grad_norm": 0.0007483125664293766,
+      "kl": 0.0007459162014129106,
+      "learning_rate": 3.8353683358814046e-07,
+      "loss": -0.0074,
+      "reward": 0.05625,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 370
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 531.85625,
+      "epoch": 0.8,
+      "grad_norm": 0.030684595927596092,
+      "kl": 0.0007768759445752948,
+      "learning_rate": 3.469460380826697e-07,
+      "loss": 0.0043,
+      "reward": 0.0375,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 375
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 516.33125,
+      "epoch": 0.8106666666666666,
+      "grad_norm": 0.017879687249660492,
+      "kl": 0.0007059701390971895,
+      "learning_rate": 3.119602417459075e-07,
+      "loss": 0.0053,
+      "reward": 0.08125,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.08125,
+      "rewards/format_reward": 0.0,
+      "step": 380
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 584.0375,
+      "epoch": 0.8213333333333334,
+      "grad_norm": 0.032094355672597885,
+      "kl": 0.0007100752409314737,
+      "learning_rate": 2.786281432302071e-07,
+      "loss": -0.0047,
+      "reward": 0.06875,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.0,
+      "step": 385
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 548.86875,
+      "epoch": 0.832,
+      "grad_norm": 0.033638887107372284,
+      "kl": 0.0007739846543699969,
+      "learning_rate": 2.46996139315057e-07,
+      "loss": 0.001,
+      "reward": 0.05625,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 390
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 557.05,
+      "epoch": 0.8426666666666667,
+      "grad_norm": 0.02078291028738022,
+      "kl": 0.0007550935217295774,
+      "learning_rate": 2.1710826032485286e-07,
+      "loss": 0.0127,
+      "reward": 0.05,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 395
+    },
+    {
+      "epoch": 0.8533333333333334,
+      "grad_norm": 0.033206239342689514,
+      "learning_rate": 1.8900610884066817e-07,
+      "loss": 0.0133,
+      "step": 400
+    },
+    {
+      "epoch": 0.8533333333333334,
+      "eval_clip_ratio": 0.0,
+      "eval_completion_length": 543.305,
+      "eval_kl": 0.0007358199717185926,
+      "eval_loss": -0.0010210861219093204,
+      "eval_reward": 0.0571,
+      "eval_reward_std": 0.06972072743177414,
+      "eval_rewards/accuracy_reward": 0.0571,
+      "eval_rewards/format_reward": 0.0,
+      "eval_runtime": 6923.7389,
+      "eval_samples_per_second": 0.722,
+      "eval_steps_per_second": 0.181,
+      "step": 400
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 553.21875,
+      "epoch": 0.864,
+      "grad_norm": 0.0289897583425045,
+      "kl": 0.000681120142689906,
+      "learning_rate": 1.627288017913383e-07,
+      "loss": 0.015,
+      "reward": 0.053125,
+      "reward_std": 0.06629125960171223,
+      "rewards/accuracy_reward": 0.053125,
+      "rewards/format_reward": 0.0,
+      "step": 405
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 541.675,
+      "epoch": 0.8746666666666667,
+      "grad_norm": 0.01583489030599594,
+      "kl": 0.0007215753896161914,
+      "learning_rate": 1.3831291600445573e-07,
+      "loss": 0.0026,
+      "reward": 0.06875,
+      "reward_std": 0.09722718074917794,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.0,
+      "step": 410
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 557.6875,
+      "epoch": 0.8853333333333333,
+      "grad_norm": 0.02964521199464798,
+      "kl": 0.0007608094070747029,
+      "learning_rate": 1.1579243729307487e-07,
+      "loss": 0.0165,
+      "reward": 0.06875,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.0,
+      "step": 415
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 565.35,
+      "epoch": 0.896,
+      "grad_norm": 0.02043159306049347,
+      "kl": 0.0006906129630806391,
+      "learning_rate": 9.519871314899092e-08,
+      "loss": 0.0003,
+      "reward": 0.075,
+      "reward_std": 0.0883883461356163,
+      "rewards/accuracy_reward": 0.075,
+      "rewards/format_reward": 0.0,
+      "step": 420
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 574.40625,
+      "epoch": 0.9066666666666666,
+      "grad_norm": 0.02876296639442444,
+      "kl": 0.0007379365182714537,
+      "learning_rate": 7.656040910844358e-08,
+      "loss": 0.0039,
+      "reward": 0.05625,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.05625,
+      "rewards/format_reward": 0.0,
+      "step": 425
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 515.95625,
+      "epoch": 0.9173333333333333,
+      "grad_norm": 0.0013636675430461764,
+      "kl": 0.0007321933764615097,
+      "learning_rate": 5.990346885098235e-08,
+      "loss": 0.002,
+      "reward": 0.05,
+      "reward_std": 0.07071067690849304,
+      "rewards/accuracy_reward": 0.05,
+      "rewards/format_reward": 0.0,
+      "step": 430
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 565.63125,
+      "epoch": 0.928,
+      "grad_norm": 0.03647277504205704,
+      "kl": 0.0007315725852095056,
+      "learning_rate": 4.5251078087033493e-08,
+      "loss": 0.0024,
+      "reward": 0.0375,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 435
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 564.30625,
+      "epoch": 0.9386666666666666,
+      "grad_norm": 0.05657452344894409,
+      "kl": 0.0007095941335137467,
+      "learning_rate": 3.262363228443427e-08,
+      "loss": -0.0054,
+      "reward": 0.04375,
+      "reward_std": 0.06187184229493141,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 440
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 486.4875,
+      "epoch": 0.9493333333333334,
+      "grad_norm": 0.01980077102780342,
+      "kl": 0.000773343399487203,
+      "learning_rate": 2.2038708278862952e-08,
+      "loss": 0.0091,
+      "reward": 0.0375,
+      "reward_std": 0.05303300768136978,
+      "rewards/accuracy_reward": 0.0375,
+      "rewards/format_reward": 0.0,
+      "step": 445
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 548.6875,
+      "epoch": 0.96,
+      "grad_norm": 0.015037409029901028,
+      "kl": 0.0007495594465581235,
+      "learning_rate": 1.3511039807673209e-08,
+      "loss": -0.0042,
+      "reward": 0.075,
+      "reward_std": 0.0883883461356163,
+      "rewards/accuracy_reward": 0.075,
+      "rewards/format_reward": 0.0,
+      "step": 450
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 558.9,
+      "epoch": 0.9706666666666667,
+      "grad_norm": 0.0009223443339578807,
+      "kl": 0.0007477809151168913,
+      "learning_rate": 7.0524970011963675e-09,
+      "loss": 0.0059,
+      "reward": 0.06875,
+      "reward_std": 0.07954951152205467,
+      "rewards/accuracy_reward": 0.06875,
+      "rewards/format_reward": 0.0,
+      "step": 455
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 579.7875,
+      "epoch": 0.9813333333333333,
+      "grad_norm": 0.008670797571539879,
+      "kl": 0.0007434627186739817,
+      "learning_rate": 2.6720698600553595e-09,
+      "loss": -0.0066,
+      "reward": 0.03125,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.03125,
+      "rewards/format_reward": 0.0,
+      "step": 460
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 582.4625,
+      "epoch": 0.992,
+      "grad_norm": 0.018711771816015244,
+      "kl": 0.0006959045043913647,
+      "learning_rate": 3.7585574148779613e-10,
+      "loss": -0.0194,
+      "reward": 0.04375,
+      "reward_std": 0.04419417306780815,
+      "rewards/accuracy_reward": 0.04375,
+      "rewards/format_reward": 0.0,
+      "step": 465
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 539.5729166666666,
+      "epoch": 0.9984,
+      "kl": 0.0007383018370698361,
+      "reward": 0.08333333333333333,
+      "reward_std": 0.058925564090410866,
+      "rewards/accuracy_reward": 0.08333333333333333,
+      "rewards/format_reward": 0.0,
+      "step": 468,
       "total_flos": 0.0,
-      "train_loss": 0.0008721697144210338,
-      "train_runtime": 2002.6709,
-      "train_samples_per_second": 0.24,
-      "train_steps_per_second": 0.015
+      "train_loss": 0.0028079951708861748,
+      "train_runtime": 39617.8147,
+      "train_samples_per_second": 0.189,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 5,
-  "max_steps": 30,
+  "max_steps": 468,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
@@ -134,7 +1391,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7ce212c5204dd486d663714791f972520601089fe064d65c6f7d1b24d7d5cb22
+oid sha256:182c7356186b1649511cf378d6ca2f74e09985432dfcecb23b9695a820bba297
 size 6520