End of training

README.md

@@ -38,7 +38,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.50.3
+- Transformers: 4.51.1
 - Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1713.5320490076206,
-    "train_runtime": 185.6881,
-    "train_samples": 160,
-    "train_samples_per_second": 1.723,
-    "train_steps_per_second": 0.108
+    "train_loss": 1.738034188747406e-06,
+    "train_runtime": 1308.1165,
+    "train_samples": 140,
+    "train_samples_per_second": 0.245,
+    "train_steps_per_second": 0.015
 }

config.json

@@ -21,7 +21,7 @@
   "sliding_window": 32768,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.50.3",
+  "transformers_version": "4.51.1",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.50.3"
+  "transformers_version": "4.51.1"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f15cdd9ca73aba6fe0a6fbd3cfcbcffd646a37d963a3c4d3ea130e46bc3e86e5
+oid sha256:a62f21bff142cf3043916bc569aa8c8f858d340bba9eb234087643c425f07556
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1713.5320490076206,
-    "train_runtime": 185.6881,
-    "train_samples": 160,
-    "train_samples_per_second": 1.723,
-    "train_steps_per_second": 0.108
+    "train_loss": 1.738034188747406e-06,
+    "train_runtime": 1308.1165,
+    "train_samples": 140,
+    "train_samples_per_second": 0.245,
+    "train_steps_per_second": 0.015
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 1.1142857142857143,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 221.125,
-      "epoch": 0.1,
-      "grad_norm": 20.139453887939453,
+      "completion_length": 320.3125,
+      "epoch": 0.11428571428571428,
+      "grad_norm": 15.526302337646484,
       "kl": 0.0,
-      "learning_rate": 4.965903258506806e-07,
+      "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 3.583044346421957,
-      "reward_std": 0.3225670934189111,
-      "rewards/concensus_correctness_reward_func": 0.8732500001788139,
-      "rewards/consensus_reward_func": 1.0,
+      "reward": 0.691326369123999,
+      "reward_std": 0.5162109876691829,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6593567873351276,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.3016076110652648,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 0.8629375072196126,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.20221875701099634,
       "step": 2
     },
     {
-      "completion_length": 218.1875,
-      "epoch": 0.2,
-      "grad_norm": 9141.5224609375,
-      "kl": 204.94856747821905,
-      "learning_rate": 4.698684378016222e-07,
-      "loss": 0.2049,
-      "reward": 5.809427872300148,
-      "reward_std": 0.40486387093551457,
-      "rewards/concensus_correctness_reward_func": 1.5327500030398369,
-      "rewards/consensus_reward_func": 1.9375,
+      "completion_length": 361.3125,
+      "epoch": 0.22857142857142856,
+      "grad_norm": 13.15450668334961,
+      "kl": 0.002233348659501644,
+      "learning_rate": 4.864543104251586e-07,
+      "loss": 0.0,
+      "reward": 0.8552711592055857,
+      "reward_std": 1.1852453968604095,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9273653849959373,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.1627399306744337,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.1618125215172768,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.057468746847007424,
       "step": 4
     },
     {
-      "completion_length": 236.0,
-      "epoch": 0.3,
-      "grad_norm": 77.85133361816406,
-      "kl": 1335.9335167701356,
-      "learning_rate": 4.193203929064353e-07,
-      "loss": 1.3359,
-      "reward": 5.69304946064949,
-      "reward_std": 0.8221923279052135,
-      "rewards/concensus_correctness_reward_func": 1.5556250112131238,
-      "rewards/consensus_reward_func": 1.75,
+      "completion_length": 396.9375,
+      "epoch": 0.34285714285714286,
+      "grad_norm": 10.020111083984375,
+      "kl": 0.0010558608701103367,
+      "learning_rate": 4.472851273490984e-07,
+      "loss": -0.0,
+      "reward": 0.6075128931552172,
+      "reward_std": 0.6859855194143165,
+      "rewards/concensus_correctness_reward_func": 0.0599374994635582,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.887486869469285,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.2983878857921809,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.140562504529953,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.17106250021606684,
       "step": 6
     },
     {
-      "completion_length": 197.375,
-      "epoch": 0.4,
-      "grad_norm": 377.45550537109375,
-      "kl": 50.22693386673927,
-      "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0502,
-      "reward": 6.55130061507225,
-      "reward_std": 0.3447979334741831,
-      "rewards/concensus_correctness_reward_func": 2.0681250169873238,
-      "rewards/consensus_reward_func": 1.75,
+      "completion_length": 345.34375,
+      "epoch": 0.45714285714285713,
+      "grad_norm": 5.887537002563477,
+      "kl": 0.001341620664788934,
+      "learning_rate": 3.867370395306068e-07,
+      "loss": 0.0,
+      "reward": 0.8436719829333015,
+      "reward_std": 0.9996534534147941,
+      "rewards/concensus_correctness_reward_func": 0.17212500050663948,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.8836442492902279,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.31995324935996905,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.1620312705636024,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.16409375239163637,
       "step": 8
     },
     {
-      "completion_length": 165.53125,
-      "epoch": 0.5,
-      "grad_norm": 1032759616.0,
-      "kl": 16418408.761649942,
-      "learning_rate": 2.706448363680831e-07,
-      "loss": 16418.4082,
-      "reward": 6.737318724393845,
-      "reward_std": 0.39243057603016496,
-      "rewards/concensus_correctness_reward_func": 2.025687500834465,
-      "rewards/consensus_reward_func": 1.9375,
+      "completion_length": 372.84375,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 7.899752616882324,
+      "kl": 0.0019128580679534934,
+      "learning_rate": 3.1137137178519977e-07,
+      "loss": 0.0,
+      "reward": 0.983705666847527,
+      "reward_std": 0.6713080538902432,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9018187988549471,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.3779244120232761,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.2160625085234642,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.293281247606501,
       "step": 10
     },
     {
-      "completion_length": 183.96875,
-      "epoch": 0.6,
-      "grad_norm": 69.0755615234375,
-      "kl": 16.420032211579382,
-      "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0164,
-      "reward": 6.181130200624466,
-      "reward_std": 0.4472991875372827,
-      "rewards/concensus_correctness_reward_func": 1.6567499972879887,
-      "rewards/consensus_reward_func": 1.875,
+      "completion_length": 373.78125,
+      "epoch": 0.6857142857142857,
+      "grad_norm": 9.6875,
+      "kl": 0.0016876287882041652,
+      "learning_rate": 2.2935516363191693e-07,
+      "loss": 0.0,
+      "reward": 0.7936931876465678,
+      "reward_std": 0.5122781188692898,
+      "rewards/concensus_correctness_reward_func": 0.06006250157952309,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9348177053034306,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3084119479171932,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.2458125054836273,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.36271873861551285,
       "step": 12
     },
     {
-      "completion_length": 184.96875,
-      "epoch": 0.7,
-      "grad_norm": 10738374.0,
-      "kl": 715220.8279513596,
-      "learning_rate": 1.1326296046939333e-07,
-      "loss": 715.2209,
-      "reward": 6.46547394990921,
-      "reward_std": 0.6154752657748759,
-      "rewards/concensus_correctness_reward_func": 1.9638124890625477,
-      "rewards/consensus_reward_func": 1.875,
+      "completion_length": 413.625,
+      "epoch": 0.8,
+      "grad_norm": 7.232576847076416,
+      "kl": 0.0023076491997926496,
+      "learning_rate": 1.4957614383675767e-07,
+      "loss": 0.0,
+      "reward": 0.7800035201944411,
+      "reward_std": 0.9252568740339484,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8580052405595779,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.24059728463180363,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.1749062612652779,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.1644062444102019,
       "step": 14
     },
     {
-      "completion_length": 183.25,
-      "epoch": 0.8,
-      "grad_norm": 66.06885528564453,
-      "kl": 42.09273688448593,
-      "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0421,
-      "reward": 6.479139924049377,
-      "reward_std": 0.39258914068341255,
-      "rewards/concensus_correctness_reward_func": 1.8927499912679195,
-      "rewards/consensus_reward_func": 1.9375,
+      "completion_length": 372.0625,
+      "epoch": 0.9142857142857143,
+      "grad_norm": 25.036296844482422,
+      "kl": 0.002074697611533338,
+      "learning_rate": 8.067960709356478e-08,
+      "loss": 0.0,
+      "reward": 0.7601466523483396,
+      "reward_std": 1.2152970017923508,
+      "rewards/concensus_correctness_reward_func": 0.1731249950826168,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.7949523320421576,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.26995914871804416,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.1976875104010105,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.0579374935477972,
       "step": 16
     },
     {
-      "completion_length": 171.3125,
-      "epoch": 0.9,
-      "grad_norm": 223.96368408203125,
-      "kl": 12.877988177351654,
-      "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0129,
-      "reward": 6.037820547819138,
-      "reward_std": 0.5467852650908753,
-      "rewards/concensus_correctness_reward_func": 1.866249993443489,
-      "rewards/consensus_reward_func": 1.75,
+      "completion_length": 294.5,
+      "epoch": 1.0,
+      "grad_norm": 5.008063316345215,
+      "kl": 0.0030747564936367175,
+      "learning_rate": 3.013156219837776e-08,
+      "loss": 0.0,
+      "reward": 0.9409237677852312,
+      "reward_std": 1.0719083348910015,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7587268259376287,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.34396544595559436,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.162843756377697,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.26362498694409925,
       "step": 18
     },
     {
-      "completion_length": 187.625,
-      "epoch": 1.0,
-      "grad_norm": 610.2742309570312,
-      "kl": 28.89718913845718,
-      "learning_rate": 0.0,
-      "loss": 0.0289,
-      "reward": 6.34273362159729,
-      "reward_std": 0.3181899096816778,
-      "rewards/concensus_correctness_reward_func": 1.8753749877214432,
-      "rewards/consensus_reward_func": 1.875,
+      "completion_length": 398.78125,
+      "epoch": 1.1142857142857143,
+      "grad_norm": 8.156458854675293,
+      "kl": 0.0040267355579999276,
+      "learning_rate": 3.4096741493194193e-09,
+      "loss": 0.0,
+      "reward": 0.5339425103738904,
+      "reward_std": 1.2569950968027115,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9278585948050022,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.31441125739365816,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.1801250167191029,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.030468733981251717,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 1.1142857142857143,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1713.5320490076206,
-      "train_runtime": 185.6881,
-      "train_samples_per_second": 1.723,
-      "train_steps_per_second": 0.108
+      "train_loss": 1.738034188747406e-06,
+      "train_runtime": 1308.1165,
+      "train_samples_per_second": 0.245,
+      "train_steps_per_second": 0.015
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce77f6e14977aab39201ed9d80237953c712a0476f7b2159056c099d7befbfa7
-size 6008
+oid sha256:c940b13e89e5826b7197d192e7ce1cb7c27a30f9db0b401fd0495f422c863410
+size 5944