Model save

README.md

@@ -1,5 +1,5 @@
 ---
-base_model: Qwen/Qwen2.5-0.5B
+base_model: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
 library_name: transformers
 model_name: Qwen-2.5-7B-Simple-RL
 tags:
@@ -11,7 +11,7 @@ licence: license
 
 # Model Card for Qwen-2.5-7B-Simple-RL
 
-This model is a fine-tuned version of [Qwen/Qwen2.5-0.5B](https://huggingface.co/Qwen/Qwen2.5-0.5B).
+This model is a fine-tuned version of [deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B](https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

adapter_config.json

@@ -1,7 +1,7 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "Qwen/Qwen2.5-0.5B",
+  "base_model_name_or_path": "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B",
   "bias": "none",
   "eva_config": null,
   "exclude_modules": null,
@@ -24,12 +24,12 @@
   "revision": null,
   "target_modules": [
     "q_proj",
-    "k_proj",
-    "gate_proj",
     "o_proj",
-    "up_proj",
     "down_proj",
-    "v_proj"
+    "k_proj",
+    "up_proj",
+    "v_proj",
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:287246a45303c5a200b655409bf837910a83e9fb80d67a180643de6dcdefbcfc
-size 35237104
+oid sha256:b398bcc7360a57c4742687ed21c8828c4e50575aa934eaeda3da7b684650e13b
+size 73911112

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0028079951708861748,
-    "train_runtime": 39617.8147,
+    "train_loss": 0.0,
+    "train_runtime": 48.8842,
     "train_samples": 7500,
-    "train_samples_per_second": 0.189,
-    "train_steps_per_second": 0.012
+    "train_samples_per_second": 0.327,
+    "train_steps_per_second": 0.02
 }

special_tokens_map.json

@@ -1,28 +1,20 @@
 {
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "eos_token": {
-    "content": "<|endoftext|>",
+    "content": "<｜end▁of▁sentence｜>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,
     "single_word": false
   },
   "pad_token": {
-    "content": "<|endoftext|>",
+    "content": "<｜end▁of▁sentence｜>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
-size 11422063
+oid sha256:a4256422650d141f228fe954acee98679da412984c29a569877eefd3af69315a
+size 11422959

tokenizer_config.json

@@ -1,9 +1,10 @@
 {
-  "add_bos_token": false,
-  "add_prefix_space": false,
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "151643": {
-      "content": "<|endoftext|>",
+      "content": "<｜end▁of▁sentence｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -11,23 +12,23 @@
       "special": true
     },
     "151644": {
-      "content": "<|im_start|>",
+      "content": "<｜User｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "151645": {
-      "content": "<|im_end|>",
+      "content": "<｜Assistant｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "151646": {
-      "content": "<|object_ref_start|>",
+      "content": "<｜begin▁of▁sentence｜>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -35,28 +36,28 @@
       "special": true
     },
     "151647": {
-      "content": "<|object_ref_end|>",
+      "content": "<|EOT|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "151648": {
-      "content": "<|box_start|>",
+      "content": "<think>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "151649": {
-      "content": "<|box_end|>",
+      "content": "</think>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "151650": {
       "content": "<|quad_start|>",
@@ -179,30 +180,16 @@
       "special": false
     }
   },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
-  "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "bos_token": "<｜begin▁of▁sentence｜>",
+  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜><think>\\n'}}{% endif %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|endoftext|>",
-  "errors": "replace",
+  "eos_token": "<｜end▁of▁sentence｜>",
   "extra_special_tokens": {},
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
+  "legacy": true,
+  "model_max_length": 16384,
+  "pad_token": "<｜end▁of▁sentence｜>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizerFast",
+  "unk_token": null,
+  "use_default_system_prompt": false
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0028079951708861748,
-    "train_runtime": 39617.8147,
+    "train_loss": 0.0,
+    "train_runtime": 48.8842,
     "train_samples": 7500,
-    "train_samples_per_second": 0.189,
-    "train_steps_per_second": 0.012
+    "train_samples_per_second": 0.327,
+    "train_steps_per_second": 0.02
 }

trainer_state.json

@@ -1,20 +1,20 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9984,
+  "epoch": 0.0010666666666666667,
   "eval_steps": 100,
-  "global_step": 468,
+  "global_step": 1,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 599.40625,
-      "epoch": 0.0021333333333333334,
+      "completion_length": 256.0,
+      "epoch": 0.0010666666666666667,
       "grad_norm": 0.0,
       "kl": 0.0,
-      "learning_rate": 6.382978723404255e-08,
+      "learning_rate": 3e-06,
       "loss": 0.0,
       "reward": 0.0,
       "reward_std": 0.0,
@@ -23,1358 +23,17 @@
       "step": 1
     },
     {
-      "clip_ratio": 0.0,
-      "completion_length": 567.0078125,
-      "epoch": 0.010666666666666666,
-      "grad_norm": 0.02547222189605236,
-      "kl": 0.00040041709871729836,
-      "learning_rate": 3.1914893617021275e-07,
-      "loss": -0.0109,
-      "reward": 0.0859375,
-      "reward_std": 0.07733980286866426,
-      "rewards/accuracy_reward": 0.0859375,
-      "rewards/format_reward": 0.0,
-      "step": 5
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 504.56875,
-      "epoch": 0.021333333333333333,
-      "grad_norm": 0.016505707055330276,
-      "kl": 0.0006629314979363698,
-      "learning_rate": 6.382978723404255e-07,
-      "loss": -0.0096,
-      "reward": 0.04375,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 10
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 505.84375,
-      "epoch": 0.032,
-      "grad_norm": 0.033047277480363846,
-      "kl": 0.0005872767855180427,
-      "learning_rate": 9.574468085106384e-07,
-      "loss": 0.0176,
-      "reward": 0.0375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 15
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 543.25625,
-      "epoch": 0.042666666666666665,
-      "grad_norm": 0.03408794477581978,
-      "kl": 0.0006242210474738386,
-      "learning_rate": 1.276595744680851e-06,
-      "loss": 0.0039,
-      "reward": 0.04375,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 20
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 597.21875,
-      "epoch": 0.05333333333333334,
-      "grad_norm": 0.02023429423570633,
-      "kl": 0.000612794059998123,
-      "learning_rate": 1.5957446808510639e-06,
-      "loss": 0.0064,
-      "reward": 0.05,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 25
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 591.35625,
-      "epoch": 0.064,
-      "grad_norm": 0.027683651074767113,
-      "kl": 0.0005714267717848998,
-      "learning_rate": 1.9148936170212767e-06,
-      "loss": -0.0076,
-      "reward": 0.05,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 30
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 516.53125,
-      "epoch": 0.07466666666666667,
-      "grad_norm": 0.0005982788279652596,
-      "kl": 0.0005893220724829007,
-      "learning_rate": 2.2340425531914894e-06,
-      "loss": 0.0301,
-      "reward": 0.08125,
-      "reward_std": 0.09722718074917794,
-      "rewards/accuracy_reward": 0.08125,
-      "rewards/format_reward": 0.0,
-      "step": 35
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 544.31875,
-      "epoch": 0.08533333333333333,
-      "grad_norm": 0.0005049636238254607,
-      "kl": 0.0006337692822853569,
-      "learning_rate": 2.553191489361702e-06,
-      "loss": 0.0217,
-      "reward": 0.08125,
-      "reward_std": 0.09722718074917794,
-      "rewards/accuracy_reward": 0.08125,
-      "rewards/format_reward": 0.0,
-      "step": 40
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 514.3875,
-      "epoch": 0.096,
-      "grad_norm": 0.03791235387325287,
-      "kl": 0.0006594170117750764,
-      "learning_rate": 2.872340425531915e-06,
-      "loss": -0.0038,
-      "reward": 0.0625,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.0625,
-      "rewards/format_reward": 0.0,
-      "step": 45
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 509.44375,
-      "epoch": 0.10666666666666667,
-      "grad_norm": 0.02021694928407669,
-      "kl": 0.0006204415323736612,
-      "learning_rate": 2.9996241442585123e-06,
-      "loss": 0.0034,
-      "reward": 0.0375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 50
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 615.1375,
-      "epoch": 0.11733333333333333,
-      "grad_norm": 0.017503464594483376,
-      "kl": 0.0005966832359263208,
-      "learning_rate": 2.9973279301399446e-06,
-      "loss": 0.0078,
-      "reward": 0.04375,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 55
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 508.95625,
-      "epoch": 0.128,
-      "grad_norm": 0.04325263574719429,
-      "kl": 0.0007028729422017932,
-      "learning_rate": 2.992947502998804e-06,
-      "loss": -0.0118,
-      "reward": 0.0625,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0625,
-      "rewards/format_reward": 0.0,
-      "step": 60
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 528.80625,
-      "epoch": 0.13866666666666666,
-      "grad_norm": 0.026006193831562996,
-      "kl": 0.0006322090062894859,
-      "learning_rate": 2.9864889601923268e-06,
-      "loss": -0.0065,
-      "reward": 0.06875,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 65
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 508.56875,
-      "epoch": 0.14933333333333335,
-      "grad_norm": 0.03273686766624451,
-      "kl": 0.0005886858463782119,
-      "learning_rate": 2.977961291721137e-06,
-      "loss": 0.0017,
-      "reward": 0.06875,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 70
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 564.89375,
-      "epoch": 0.16,
-      "grad_norm": 0.012765306048095226,
-      "kl": 0.0005601999211648945,
-      "learning_rate": 2.9673763677155655e-06,
-      "loss": -0.0157,
-      "reward": 0.05,
-      "reward_std": 0.03535533845424652,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 75
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 560.225,
-      "epoch": 0.17066666666666666,
-      "grad_norm": 0.023025022819638252,
-      "kl": 0.0007396811590297148,
-      "learning_rate": 2.9547489219129666e-06,
-      "loss": 0.005,
-      "reward": 0.04375,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 80
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 534.0,
-      "epoch": 0.18133333333333335,
-      "grad_norm": 0.0005447011208161712,
-      "kl": 0.0005739938373153563,
-      "learning_rate": 2.9400965311490175e-06,
-      "loss": 0.0065,
-      "reward": 0.025,
-      "reward_std": 0.03535533845424652,
-      "rewards/accuracy_reward": 0.025,
-      "rewards/format_reward": 0.0,
-      "step": 85
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 578.45625,
-      "epoch": 0.192,
-      "grad_norm": 0.05320625752210617,
-      "kl": 0.0005843402541358955,
-      "learning_rate": 2.9234395908915565e-06,
-      "loss": 0.0148,
-      "reward": 0.05625,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 90
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 529.00625,
-      "epoch": 0.20266666666666666,
-      "grad_norm": 0.0005686267395503819,
-      "kl": 0.0006063115164579358,
-      "learning_rate": 2.904801286851009e-06,
-      "loss": -0.0111,
-      "reward": 0.025,
-      "reward_std": 0.03535533845424652,
-      "rewards/accuracy_reward": 0.025,
-      "rewards/format_reward": 0.0,
-      "step": 95
-    },
-    {
-      "epoch": 0.21333333333333335,
-      "grad_norm": 0.0004754478286486119,
-      "learning_rate": 2.884207562706925e-06,
-      "loss": -0.0045,
-      "step": 100
-    },
-    {
-      "epoch": 0.21333333333333335,
-      "eval_clip_ratio": 0.0,
-      "eval_completion_length": 546.1873,
-      "eval_kl": 0.0006224736573698465,
-      "eval_loss": 0.0016505873063579202,
-      "eval_reward": 0.0431,
-      "eval_reward_std": 0.05246732226610184,
-      "eval_rewards/accuracy_reward": 0.0431,
-      "eval_rewards/format_reward": 0.0,
-      "eval_runtime": 6984.4091,
-      "eval_samples_per_second": 0.716,
-      "eval_steps_per_second": 0.179,
-      "step": 100
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 504.8,
-      "epoch": 0.224,
-      "grad_norm": 0.043887630105018616,
-      "kl": 0.0005905847807298415,
-      "learning_rate": 2.8616870839955444e-06,
-      "loss": -0.0069,
-      "reward": 0.065625,
-      "reward_std": 0.08396892882883548,
-      "rewards/accuracy_reward": 0.065625,
-      "rewards/format_reward": 0.0,
-      "step": 105
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 554.9875,
-      "epoch": 0.23466666666666666,
-      "grad_norm": 0.03810940682888031,
-      "kl": 0.0006392004950612317,
-      "learning_rate": 2.837271198208662e-06,
-      "loss": 0.0054,
-      "reward": 0.05625,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 110
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 594.39375,
-      "epoch": 0.24533333333333332,
-      "grad_norm": 0.0005416510975919664,
-      "kl": 0.0006476349917647894,
-      "learning_rate": 2.8109938911593322e-06,
-      "loss": -0.0032,
-      "reward": 0.00625,
-      "reward_std": 0.00883883461356163,
-      "rewards/accuracy_reward": 0.00625,
-      "rewards/format_reward": 0.0,
-      "step": 115
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 535.65625,
-      "epoch": 0.256,
-      "grad_norm": 0.021277422085404396,
-      "kl": 0.000646474702807609,
-      "learning_rate": 2.7828917396751474e-06,
-      "loss": 0.0019,
-      "reward": 0.05,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 120
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 497.81875,
-      "epoch": 0.26666666666666666,
-      "grad_norm": 0.04059675708413124,
-      "kl": 0.0007086705743859056,
-      "learning_rate": 2.753003860684943e-06,
-      "loss": 0.0073,
-      "reward": 0.03125,
-      "reward_std": 0.02651650384068489,
-      "rewards/accuracy_reward": 0.03125,
-      "rewards/format_reward": 0.0,
-      "step": 125
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 539.66875,
-      "epoch": 0.2773333333333333,
-      "grad_norm": 0.0311752837151289,
-      "kl": 0.0006064358283765614,
-      "learning_rate": 2.721371856769793e-06,
-      "loss": 0.0056,
-      "reward": 0.04375,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 130
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 461.6625,
-      "epoch": 0.288,
-      "grad_norm": 0.018207967281341553,
-      "kl": 0.0006391550123225898,
-      "learning_rate": 2.688039758254093e-06,
-      "loss": 0.0073,
-      "reward": 0.06875,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 135
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 555.3,
-      "epoch": 0.2986666666666667,
-      "grad_norm": 0.014577426947653294,
-      "kl": 0.0006190024098032155,
-      "learning_rate": 2.65305396191733e-06,
-      "loss": 0.0091,
-      "reward": 0.03125,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.03125,
-      "rewards/format_reward": 0.0,
-      "step": 140
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 529.79375,
-      "epoch": 0.30933333333333335,
-      "grad_norm": 0.0005597746348939836,
-      "kl": 0.0007494706653233152,
-      "learning_rate": 2.61646316641186e-06,
-      "loss": 0.0059,
-      "reward": 0.05625,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 145
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 533.4125,
-      "epoch": 0.32,
-      "grad_norm": 0.0008026298601180315,
-      "kl": 0.000564340456185164,
-      "learning_rate": 2.5783183044765715e-06,
-      "loss": 0.0155,
-      "reward": 0.04375,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 150
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 572.88125,
-      "epoch": 0.33066666666666666,
-      "grad_norm": 0.02461259439587593,
-      "kl": 0.0006232189080037642,
-      "learning_rate": 2.5386724720408135e-06,
-      "loss": 0.0042,
-      "reward": 0.0625,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0625,
-      "rewards/format_reward": 0.0,
-      "step": 155
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 469.73125,
-      "epoch": 0.3413333333333333,
-      "grad_norm": 0.0007159389788284898,
-      "kl": 0.0006267319105973002,
-      "learning_rate": 2.49758085431725e-06,
-      "loss": -0.0023,
-      "reward": 0.0625,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0625,
-      "rewards/format_reward": 0.0,
-      "step": 160
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 537.625,
-      "epoch": 0.352,
-      "grad_norm": 0.0005227501387707889,
-      "kl": 0.0006310752982244594,
-      "learning_rate": 2.455100648986533e-06,
-      "loss": 0.0019,
-      "reward": 0.05,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 165
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 570.20625,
-      "epoch": 0.3626666666666667,
-      "grad_norm": 0.0317506417632103,
-      "kl": 0.0006065379689971451,
-      "learning_rate": 2.4112909865807053e-06,
-      "loss": 0.0383,
-      "reward": 0.08125,
-      "reward_std": 0.1149048499763012,
-      "rewards/accuracy_reward": 0.08125,
-      "rewards/format_reward": 0.0,
-      "step": 170
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 548.31875,
-      "epoch": 0.37333333333333335,
-      "grad_norm": 0.04262116923928261,
-      "kl": 0.0005954368047241587,
-      "learning_rate": 2.366212848176164e-06,
-      "loss": -0.0139,
-      "reward": 0.08125,
-      "reward_std": 0.09722718074917794,
-      "rewards/accuracy_reward": 0.08125,
-      "rewards/format_reward": 0.0,
-      "step": 175
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 506.475,
-      "epoch": 0.384,
-      "grad_norm": 0.029712267220020294,
-      "kl": 0.0006351386713504325,
-      "learning_rate": 2.319928980510752e-06,
-      "loss": -0.0088,
-      "reward": 0.06875,
-      "reward_std": 0.09722718074917794,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 180
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 553.3,
-      "epoch": 0.39466666666666667,
-      "grad_norm": 0.01826683059334755,
-      "kl": 0.000638010091643082,
-      "learning_rate": 2.272503808643123e-06,
-      "loss": -0.0008,
-      "reward": 0.04375,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 185
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 572.2125,
-      "epoch": 0.4053333333333333,
-      "grad_norm": 0.025073450058698654,
-      "kl": 0.0006133078852144536,
-      "learning_rate": 2.2240033462759628e-06,
-      "loss": 0.0062,
-      "reward": 0.0625,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.0625,
-      "rewards/format_reward": 0.0,
-      "step": 190
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 529.24375,
-      "epoch": 0.416,
-      "grad_norm": 0.017348134890198708,
-      "kl": 0.0006518879818031564,
-      "learning_rate": 2.1744951038678905e-06,
-      "loss": 0.0129,
-      "reward": 0.0875,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0875,
-      "rewards/format_reward": 0.0,
-      "step": 195
-    },
-    {
-      "epoch": 0.4266666666666667,
-      "grad_norm": 0.0005302979261614382,
-      "learning_rate": 2.124047994661941e-06,
-      "loss": -0.0062,
-      "step": 200
-    },
-    {
-      "epoch": 0.4266666666666667,
-      "eval_clip_ratio": 0.0,
-      "eval_completion_length": 542.9258,
-      "eval_kl": 0.0006711028324672952,
-      "eval_loss": 0.0005841926322318614,
-      "eval_reward": 0.0475,
-      "eval_reward_std": 0.05670996288061142,
-      "eval_rewards/accuracy_reward": 0.0475,
-      "eval_rewards/format_reward": 0.0,
-      "eval_runtime": 6903.4955,
-      "eval_samples_per_second": 0.724,
-      "eval_steps_per_second": 0.181,
-      "step": 200
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 571.925,
-      "epoch": 0.43733333333333335,
-      "grad_norm": 0.03621024638414383,
-      "kl": 0.000652521890879143,
-      "learning_rate": 2.072732238761434e-06,
-      "loss": 0.0023,
-      "reward": 0.04375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 205
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 556.3875,
-      "epoch": 0.448,
-      "grad_norm": 0.01959877274930477,
-      "kl": 0.0006465273501817137,
-      "learning_rate": 2.0206192653867536e-06,
-      "loss": -0.0093,
-      "reward": 0.04375,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 210
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 524.30625,
-      "epoch": 0.45866666666666667,
-      "grad_norm": 0.026872603222727776,
-      "kl": 0.0006922339722223114,
-      "learning_rate": 1.967781613449095e-06,
-      "loss": 0.0152,
-      "reward": 0.0625,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0625,
-      "rewards/format_reward": 0.0,
-      "step": 215
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 524.2,
-      "epoch": 0.4693333333333333,
-      "grad_norm": 0.024317504838109016,
-      "kl": 0.0007104755510226824,
-      "learning_rate": 1.9142928305795637e-06,
-      "loss": 0.005,
-      "reward": 0.0375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 220
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 566.8,
-      "epoch": 0.48,
-      "grad_norm": 0.0005844333791173995,
-      "kl": 0.0007887360618042294,
-      "learning_rate": 1.8602273707541886e-06,
-      "loss": -0.004,
-      "reward": 0.01875,
-      "reward_std": 0.02651650384068489,
-      "rewards/accuracy_reward": 0.01875,
-      "rewards/format_reward": 0.0,
-      "step": 225
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 566.1625,
-      "epoch": 0.49066666666666664,
-      "grad_norm": 0.042061224579811096,
-      "kl": 0.0006022223657055292,
-      "learning_rate": 1.8056604906573418e-06,
-      "loss": 0.0171,
-      "reward": 0.05625,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 230
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 589.93125,
-      "epoch": 0.5013333333333333,
-      "grad_norm": 0.017109017819166183,
-      "kl": 0.0006615983191295526,
-      "learning_rate": 1.7506681449278226e-06,
-      "loss": 0.0003,
-      "reward": 0.05625,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 235
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 582.16875,
-      "epoch": 0.512,
-      "grad_norm": 0.02843085303902626,
-      "kl": 0.0006830113568867091,
-      "learning_rate": 1.6953268804334257e-06,
-      "loss": 0.008,
-      "reward": 0.03125,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.03125,
-      "rewards/format_reward": 0.0,
-      "step": 240
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 550.20625,
-      "epoch": 0.5226666666666666,
-      "grad_norm": 0.0007241690182127059,
-      "kl": 0.0007094932108884678,
-      "learning_rate": 1.6397137297211436e-06,
-      "loss": 0.0047,
-      "reward": 0.03125,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.03125,
-      "rewards/format_reward": 0.0,
-      "step": 245
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 520.5375,
-      "epoch": 0.5333333333333333,
-      "grad_norm": 0.013676224276423454,
-      "kl": 0.0006993950722971931,
-      "learning_rate": 1.5839061037913395e-06,
-      "loss": -0.0035,
-      "reward": 0.0375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 250
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 570.4625,
-      "epoch": 0.544,
-      "grad_norm": 0.026784850284457207,
-      "kl": 0.0007027815772744361,
-      "learning_rate": 1.527981684345115e-06,
-      "loss": 0.0115,
-      "reward": 0.05625,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 255
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 526.28125,
-      "epoch": 0.5546666666666666,
-      "grad_norm": 0.027511239051818848,
-      "kl": 0.0007118464120139834,
-      "learning_rate": 1.4720183156548855e-06,
-      "loss": 0.0156,
-      "reward": 0.05625,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 260
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 501.05,
-      "epoch": 0.5653333333333334,
-      "grad_norm": 0.0400506816804409,
-      "kl": 0.0007247071487654466,
-      "learning_rate": 1.4160938962086612e-06,
-      "loss": 0.0056,
-      "reward": 0.05,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 265
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 492.38125,
-      "epoch": 0.576,
-      "grad_norm": 0.020546777173876762,
-      "kl": 0.0006569563163793645,
-      "learning_rate": 1.3602862702788567e-06,
-      "loss": -0.0087,
-      "reward": 0.06875,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 270
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 601.91875,
-      "epoch": 0.5866666666666667,
-      "grad_norm": 0.0006509393570013344,
-      "kl": 0.000718773623520974,
-      "learning_rate": 1.3046731195665748e-06,
-      "loss": -0.0019,
-      "reward": 0.03125,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.03125,
-      "rewards/format_reward": 0.0,
-      "step": 275
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 544.3875,
-      "epoch": 0.5973333333333334,
-      "grad_norm": 0.015962699428200722,
-      "kl": 0.0007544517939095385,
-      "learning_rate": 1.2493318550721775e-06,
-      "loss": 0.0026,
-      "reward": 0.0375,
-      "reward_std": 0.03535533845424652,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 280
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 513.525,
-      "epoch": 0.608,
-      "grad_norm": 0.02608601376414299,
-      "kl": 0.0006703489023493602,
-      "learning_rate": 1.1943395093426585e-06,
-      "loss": -0.0063,
-      "reward": 0.05625,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 285
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 537.90625,
-      "epoch": 0.6186666666666667,
-      "grad_norm": 0.0007309815846383572,
-      "kl": 0.0008104784224997275,
-      "learning_rate": 1.1397726292458115e-06,
-      "loss": 0.0138,
-      "reward": 0.04375,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 290
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 530.6,
-      "epoch": 0.6293333333333333,
-      "grad_norm": 0.034363504499197006,
-      "kl": 0.0007202902772405651,
-      "learning_rate": 1.085707169420437e-06,
-      "loss": 0.0225,
-      "reward": 0.04375,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 295
-    },
-    {
-      "epoch": 0.64,
-      "grad_norm": 0.03348564729094505,
-      "learning_rate": 1.0322183865509054e-06,
-      "loss": 0.0041,
-      "step": 300
-    },
-    {
-      "epoch": 0.64,
-      "eval_clip_ratio": 0.0,
-      "eval_completion_length": 543.87,
-      "eval_kl": 0.0007330777346272952,
-      "eval_loss": 0.002262996742501855,
-      "eval_reward": 0.0493,
-      "eval_reward_std": 0.05840701912641525,
-      "eval_rewards/accuracy_reward": 0.0493,
-      "eval_rewards/format_reward": 0.0,
-      "eval_runtime": 6924.8337,
-      "eval_samples_per_second": 0.722,
-      "eval_steps_per_second": 0.181,
-      "step": 300
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 553.821875,
-      "epoch": 0.6506666666666666,
-      "grad_norm": 0.05884533375501633,
-      "kl": 0.0007077409793055268,
-      "learning_rate": 9.793807346132464e-07,
-      "loss": 0.0064,
-      "reward": 0.065625,
-      "reward_std": 0.09280776344239712,
-      "rewards/accuracy_reward": 0.065625,
-      "rewards/format_reward": 0.0,
-      "step": 305
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 507.25625,
-      "epoch": 0.6613333333333333,
-      "grad_norm": 0.00128034723456949,
-      "kl": 0.0007662738556973636,
-      "learning_rate": 9.272677612385667e-07,
-      "loss": -0.0022,
-      "reward": 0.05625,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 310
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 585.83125,
-      "epoch": 0.672,
-      "grad_norm": 0.027511116117239,
-      "kl": 0.0007258923047629651,
-      "learning_rate": 8.759520053380591e-07,
-      "loss": 0.0104,
-      "reward": 0.0375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 315
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 566.825,
-      "epoch": 0.6826666666666666,
-      "grad_norm": 0.02675638347864151,
-      "kl": 0.0008431313741311897,
-      "learning_rate": 8.255048961321088e-07,
-      "loss": 0.0081,
-      "reward": 0.0375,
-      "reward_std": 0.03535533845424652,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 320
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 561.75,
-      "epoch": 0.6933333333333334,
-      "grad_norm": 0.03926326334476471,
-      "kl": 0.0007520209066569806,
-      "learning_rate": 7.759966537240373e-07,
-      "loss": -0.0255,
-      "reward": 0.09375,
-      "reward_std": 0.09722718074917794,
-      "rewards/accuracy_reward": 0.09375,
-      "rewards/format_reward": 0.0,
-      "step": 325
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 602.11875,
-      "epoch": 0.704,
-      "grad_norm": 0.04235618934035301,
-      "kl": 0.0006708401182550005,
-      "learning_rate": 7.274961913568773e-07,
-      "loss": -0.0017,
-      "reward": 0.0625,
-      "reward_std": 0.0883883461356163,
-      "rewards/accuracy_reward": 0.0625,
-      "rewards/format_reward": 0.0,
-      "step": 330
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 539.75625,
-      "epoch": 0.7146666666666667,
-      "grad_norm": 0.01949181966483593,
-      "kl": 0.0007214361037767957,
-      "learning_rate": 6.800710194892484e-07,
-      "loss": -0.0142,
-      "reward": 0.05,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 335
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 559.64375,
-      "epoch": 0.7253333333333334,
-      "grad_norm": 0.01706821843981743,
-      "kl": 0.0007962717980262823,
-      "learning_rate": 6.33787151823836e-07,
-      "loss": 0.0083,
-      "reward": 0.05,
-      "reward_std": 0.03535533845424652,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 340
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 543.775,
-      "epoch": 0.736,
-      "grad_norm": 0.026210952550172806,
-      "kl": 0.000737834313622443,
-      "learning_rate": 5.887090134192947e-07,
-      "loss": 0.0213,
-      "reward": 0.05625,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 345
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 512.84375,
-      "epoch": 0.7466666666666667,
-      "grad_norm": 0.026430701836943626,
-      "kl": 0.0007904948972282,
-      "learning_rate": 5.448993510134669e-07,
-      "loss": -0.0074,
-      "reward": 0.03125,
-      "reward_std": 0.02651650384068489,
-      "rewards/accuracy_reward": 0.03125,
-      "rewards/format_reward": 0.0,
-      "step": 350
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 498.09375,
-      "epoch": 0.7573333333333333,
-      "grad_norm": 0.039444226771593094,
-      "kl": 0.0007300119003048166,
-      "learning_rate": 5.024191456827498e-07,
-      "loss": 0.0032,
-      "reward": 0.05,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 355
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 534.31875,
-      "epoch": 0.768,
-      "grad_norm": 0.038000140339136124,
-      "kl": 0.000727705340977991,
-      "learning_rate": 4.6132752795918667e-07,
-      "loss": -0.0045,
-      "reward": 0.05,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 360
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 554.78125,
-      "epoch": 0.7786666666666666,
-      "grad_norm": 0.04807225242257118,
-      "kl": 0.0009294274394051171,
-      "learning_rate": 4.2168169552342905e-07,
-      "loss": -0.0011,
-      "reward": 0.05,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 365
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 552.01875,
-      "epoch": 0.7893333333333333,
-      "grad_norm": 0.0007483125664293766,
-      "kl": 0.0007459162014129106,
-      "learning_rate": 3.8353683358814046e-07,
-      "loss": -0.0074,
-      "reward": 0.05625,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 370
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 531.85625,
-      "epoch": 0.8,
-      "grad_norm": 0.030684595927596092,
-      "kl": 0.0007768759445752948,
-      "learning_rate": 3.469460380826697e-07,
-      "loss": 0.0043,
-      "reward": 0.0375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 375
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 516.33125,
-      "epoch": 0.8106666666666666,
-      "grad_norm": 0.017879687249660492,
-      "kl": 0.0007059701390971895,
-      "learning_rate": 3.119602417459075e-07,
-      "loss": 0.0053,
-      "reward": 0.08125,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.08125,
-      "rewards/format_reward": 0.0,
-      "step": 380
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 584.0375,
-      "epoch": 0.8213333333333334,
-      "grad_norm": 0.032094355672597885,
-      "kl": 0.0007100752409314737,
-      "learning_rate": 2.786281432302071e-07,
-      "loss": -0.0047,
-      "reward": 0.06875,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 385
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 548.86875,
-      "epoch": 0.832,
-      "grad_norm": 0.033638887107372284,
-      "kl": 0.0007739846543699969,
-      "learning_rate": 2.46996139315057e-07,
-      "loss": 0.001,
-      "reward": 0.05625,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 390
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 557.05,
-      "epoch": 0.8426666666666667,
-      "grad_norm": 0.02078291028738022,
-      "kl": 0.0007550935217295774,
-      "learning_rate": 2.1710826032485286e-07,
-      "loss": 0.0127,
-      "reward": 0.05,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 395
-    },
-    {
-      "epoch": 0.8533333333333334,
-      "grad_norm": 0.033206239342689514,
-      "learning_rate": 1.8900610884066817e-07,
-      "loss": 0.0133,
-      "step": 400
-    },
-    {
-      "epoch": 0.8533333333333334,
-      "eval_clip_ratio": 0.0,
-      "eval_completion_length": 543.305,
-      "eval_kl": 0.0007358199717185926,
-      "eval_loss": -0.0010210861219093204,
-      "eval_reward": 0.0571,
-      "eval_reward_std": 0.06972072743177414,
-      "eval_rewards/accuracy_reward": 0.0571,
-      "eval_rewards/format_reward": 0.0,
-      "eval_runtime": 6923.7389,
-      "eval_samples_per_second": 0.722,
-      "eval_steps_per_second": 0.181,
-      "step": 400
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 553.21875,
-      "epoch": 0.864,
-      "grad_norm": 0.0289897583425045,
-      "kl": 0.000681120142689906,
-      "learning_rate": 1.627288017913383e-07,
-      "loss": 0.015,
-      "reward": 0.053125,
-      "reward_std": 0.06629125960171223,
-      "rewards/accuracy_reward": 0.053125,
-      "rewards/format_reward": 0.0,
-      "step": 405
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 541.675,
-      "epoch": 0.8746666666666667,
-      "grad_norm": 0.01583489030599594,
-      "kl": 0.0007215753896161914,
-      "learning_rate": 1.3831291600445573e-07,
-      "loss": 0.0026,
-      "reward": 0.06875,
-      "reward_std": 0.09722718074917794,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 410
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 557.6875,
-      "epoch": 0.8853333333333333,
-      "grad_norm": 0.02964521199464798,
-      "kl": 0.0007608094070747029,
-      "learning_rate": 1.1579243729307487e-07,
-      "loss": 0.0165,
-      "reward": 0.06875,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 415
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 565.35,
-      "epoch": 0.896,
-      "grad_norm": 0.02043159306049347,
-      "kl": 0.0006906129630806391,
-      "learning_rate": 9.519871314899092e-08,
-      "loss": 0.0003,
-      "reward": 0.075,
-      "reward_std": 0.0883883461356163,
-      "rewards/accuracy_reward": 0.075,
-      "rewards/format_reward": 0.0,
-      "step": 420
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 574.40625,
-      "epoch": 0.9066666666666666,
-      "grad_norm": 0.02876296639442444,
-      "kl": 0.0007379365182714537,
-      "learning_rate": 7.656040910844358e-08,
-      "loss": 0.0039,
-      "reward": 0.05625,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.05625,
-      "rewards/format_reward": 0.0,
-      "step": 425
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 515.95625,
-      "epoch": 0.9173333333333333,
-      "grad_norm": 0.0013636675430461764,
-      "kl": 0.0007321933764615097,
-      "learning_rate": 5.990346885098235e-08,
-      "loss": 0.002,
-      "reward": 0.05,
-      "reward_std": 0.07071067690849304,
-      "rewards/accuracy_reward": 0.05,
-      "rewards/format_reward": 0.0,
-      "step": 430
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 565.63125,
-      "epoch": 0.928,
-      "grad_norm": 0.03647277504205704,
-      "kl": 0.0007315725852095056,
-      "learning_rate": 4.5251078087033493e-08,
-      "loss": 0.0024,
-      "reward": 0.0375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 435
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 564.30625,
-      "epoch": 0.9386666666666666,
-      "grad_norm": 0.05657452344894409,
-      "kl": 0.0007095941335137467,
-      "learning_rate": 3.262363228443427e-08,
-      "loss": -0.0054,
-      "reward": 0.04375,
-      "reward_std": 0.06187184229493141,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 440
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 486.4875,
-      "epoch": 0.9493333333333334,
-      "grad_norm": 0.01980077102780342,
-      "kl": 0.000773343399487203,
-      "learning_rate": 2.2038708278862952e-08,
-      "loss": 0.0091,
-      "reward": 0.0375,
-      "reward_std": 0.05303300768136978,
-      "rewards/accuracy_reward": 0.0375,
-      "rewards/format_reward": 0.0,
-      "step": 445
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 548.6875,
-      "epoch": 0.96,
-      "grad_norm": 0.015037409029901028,
-      "kl": 0.0007495594465581235,
-      "learning_rate": 1.3511039807673209e-08,
-      "loss": -0.0042,
-      "reward": 0.075,
-      "reward_std": 0.0883883461356163,
-      "rewards/accuracy_reward": 0.075,
-      "rewards/format_reward": 0.0,
-      "step": 450
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 558.9,
-      "epoch": 0.9706666666666667,
-      "grad_norm": 0.0009223443339578807,
-      "kl": 0.0007477809151168913,
-      "learning_rate": 7.0524970011963675e-09,
-      "loss": 0.0059,
-      "reward": 0.06875,
-      "reward_std": 0.07954951152205467,
-      "rewards/accuracy_reward": 0.06875,
-      "rewards/format_reward": 0.0,
-      "step": 455
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 579.7875,
-      "epoch": 0.9813333333333333,
-      "grad_norm": 0.008670797571539879,
-      "kl": 0.0007434627186739817,
-      "learning_rate": 2.6720698600553595e-09,
-      "loss": -0.0066,
-      "reward": 0.03125,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.03125,
-      "rewards/format_reward": 0.0,
-      "step": 460
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 582.4625,
-      "epoch": 0.992,
-      "grad_norm": 0.018711771816015244,
-      "kl": 0.0006959045043913647,
-      "learning_rate": 3.7585574148779613e-10,
-      "loss": -0.0194,
-      "reward": 0.04375,
-      "reward_std": 0.04419417306780815,
-      "rewards/accuracy_reward": 0.04375,
-      "rewards/format_reward": 0.0,
-      "step": 465
-    },
-    {
-      "clip_ratio": 0.0,
-      "completion_length": 539.5729166666666,
-      "epoch": 0.9984,
-      "kl": 0.0007383018370698361,
-      "reward": 0.08333333333333333,
-      "reward_std": 0.058925564090410866,
-      "rewards/accuracy_reward": 0.08333333333333333,
-      "rewards/format_reward": 0.0,
-      "step": 468,
+      "epoch": 0.0010666666666666667,
+      "step": 1,
       "total_flos": 0.0,
-      "train_loss": 0.0028079951708861748,
-      "train_runtime": 39617.8147,
-      "train_samples_per_second": 0.189,
-      "train_steps_per_second": 0.012
+      "train_loss": 0.0,
+      "train_runtime": 48.8842,
+      "train_samples_per_second": 0.327,
+      "train_steps_per_second": 0.02
     }
   ],
   "logging_steps": 5,
-  "max_steps": 468,
+  "max_steps": 1,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 500,
@@ -1391,7 +50,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 4,
+  "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:182c7356186b1649511cf378d6ca2f74e09985432dfcecb23b9695a820bba297
-size 6520
+oid sha256:e5b194307e6c24945ff1ba49c49cffa44544d0fe79d6516f1c19296273677ed1
+size 6584