End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 818.6132557076751,
-    "train_runtime": 193.9967,
+    "train_loss": 38319.64781138357,
+    "train_runtime": 193.4127,
     "train_samples": 160,
-    "train_samples_per_second": 1.65,
+    "train_samples_per_second": 1.654,
     "train_steps_per_second": 0.103
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4e095d2f1fd8ba9828b63cd5db4333514d99e277b3bc9d54cd6bcc602c36c908
+oid sha256:7dbe3f92d3e8e56ea6129db6c3d433579cc4895b85e9fccc3e679ac4f6206ed0
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 818.6132557076751,
-    "train_runtime": 193.9967,
+    "train_loss": 38319.64781138357,
+    "train_runtime": 193.4127,
     "train_samples": 160,
-    "train_samples_per_second": 1.65,
+    "train_samples_per_second": 1.654,
     "train_steps_per_second": 0.103
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 315.4375,
+      "completion_length": 268.125,
       "epoch": 0.1,
-      "grad_norm": 39.493831634521484,
+      "grad_norm": 52.51573181152344,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 4.000197161920369,
-      "reward_std": 0.6450665253214538,
-      "rewards/concensus_correctness_reward_func": 1.1254374869167805,
-      "rewards/consensus_reward_func": 1.1875,
+      "reward": 3.7880874909460545,
+      "reward_std": 0.7253511965391226,
+      "rewards/concensus_correctness_reward_func": 0.8900625109672546,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6665408746339381,
+      "rewards/question_recreation_reward_func": 0.5797749441117048,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 0.770718758678413,
+      "rewards/strict_format_reward_func": 0.21875,
+      "rewards/xmlcount_reward_func": 0.9120000023394823,
       "step": 2
     },
     {
-      "completion_length": 203.4375,
+      "completion_length": 207.125,
       "epoch": 0.2,
-      "grad_norm": 496.6695556640625,
-      "kl": 12.846051003783941,
+      "grad_norm": 14097.90234375,
+      "kl": 530.8586953068152,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0128,
-      "reward": 5.950691074132919,
-      "reward_std": 0.5357042026589625,
-      "rewards/concensus_correctness_reward_func": 1.5871250107884407,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.5309,
+      "reward": 5.9291402995586395,
+      "reward_std": 0.2267392118810676,
+      "rewards/concensus_correctness_reward_func": 1.6063750013709068,
+      "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.9265973679721355,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.9300465881824493,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1713437661528587,
+      "rewards/strict_format_reward_func": 0.21875,
+      "rewards/xmlcount_reward_func": 1.1739687621593475,
       "step": 4
     },
     {
-      "completion_length": 214.75,
+      "completion_length": 213.8125,
       "epoch": 0.3,
-      "grad_norm": 92421.5703125,
-      "kl": 7853794.46428321,
+      "grad_norm": 1793.24462890625,
+      "kl": 1615.383490987122,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 7853.7939,
-      "reward": 6.078612729907036,
-      "reward_std": 0.47438807651633397,
-      "rewards/concensus_correctness_reward_func": 1.598312510177493,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 1.6154,
+      "reward": 6.197655111551285,
+      "reward_std": 0.5888944863327197,
+      "rewards/concensus_correctness_reward_func": 1.7123124990612268,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.9641127623617649,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9221238419413567,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.2036875039339066,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.2038437500596046,
       "step": 6
     },
     {
-      "completion_length": 216.78125,
+      "completion_length": 237.3125,
       "epoch": 0.4,
-      "grad_norm": 4662240.0,
-      "kl": 277950.34182211943,
+      "grad_norm": 2184983.5,
+      "kl": 80967.61272667907,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 277.9503,
-      "reward": 6.177099883556366,
-      "reward_std": 0.39557756669819355,
-      "rewards/concensus_correctness_reward_func": 1.6980624869465828,
+      "loss": 80.9676,
+      "reward": 6.5779251754283905,
+      "reward_std": 0.5500927083194256,
+      "rewards/concensus_correctness_reward_func": 1.9547500163316727,
       "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9542873911559582,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.9303939752280712,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1966250017285347,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.1927812546491623,
       "step": 8
     },
     {
-      "completion_length": 186.15625,
+      "completion_length": 184.125,
       "epoch": 0.5,
-      "grad_norm": 737.678466796875,
-      "kl": 189.93280582735315,
+      "grad_norm": 28116684800.0,
+      "kl": 378830514.4991078,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.1899,
-      "reward": 6.838829427957535,
-      "reward_std": 0.3894164815574186,
-      "rewards/concensus_correctness_reward_func": 2.023374993354082,
+      "loss": 378830.5938,
+      "reward": 6.973221987485886,
+      "reward_std": 0.4501988561823964,
+      "rewards/concensus_correctness_reward_func": 2.1776249930262566,
       "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9017982184886932,
+      "rewards/final_correctness_reward_func": 0.5625,
+      "rewards/question_recreation_reward_func": 0.8454405581578612,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.2417812570929527,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.1845312640070915,
       "step": 10
     },
     {
-      "completion_length": 214.09375,
+      "completion_length": 220.78125,
       "epoch": 0.6,
-      "grad_norm": 33949.3828125,
-      "kl": 1872.4105841587298,
+      "grad_norm": 538161.25,
+      "kl": 36245.77570069302,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 1.8724,
-      "reward": 5.980763286352158,
-      "reward_std": 0.38482404364913236,
-      "rewards/concensus_correctness_reward_func": 1.5693749897181988,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 36.2458,
+      "reward": 6.162264287471771,
+      "reward_std": 0.28252823650836945,
+      "rewards/concensus_correctness_reward_func": 1.7267500013113022,
+      "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9559195451438427,
+      "rewards/question_recreation_reward_func": 0.8918579369783401,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.2210937440395355,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 1.2311562672257423,
       "step": 12
     },
     {
-      "completion_length": 184.96875,
+      "completion_length": 202.78125,
       "epoch": 0.7,
-      "grad_norm": 1177.0413818359375,
-      "kl": 23.387942592264153,
+      "grad_norm": 1045.7855224609375,
+      "kl": 208.7559353262186,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0234,
-      "reward": 6.239442318677902,
-      "reward_std": 0.2518596774316393,
-      "rewards/concensus_correctness_reward_func": 1.7816874980926514,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.2088,
+      "reward": 6.0759420692920685,
+      "reward_std": 0.5171458608820103,
+      "rewards/concensus_correctness_reward_func": 1.8015624918043613,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8934109658002853,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.8865983616560698,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.2205937504768372,
+      "rewards/strict_format_reward_func": 0.21875,
+      "rewards/xmlcount_reward_func": 1.1690312623977661,
       "step": 14
     },
     {
-      "completion_length": 202.5625,
+      "completion_length": 200.25,
       "epoch": 0.8,
-      "grad_norm": 40.06753158569336,
-      "kl": 279.377266407595,
+      "grad_norm": 108.79785919189453,
+      "kl": 216.4876526929438,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.2794,
-      "reward": 6.441562205553055,
-      "reward_std": 0.47932186246907804,
-      "rewards/concensus_correctness_reward_func": 1.9303749948740005,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.2165,
+      "reward": 6.525058060884476,
+      "reward_std": 0.9049128945916891,
+      "rewards/concensus_correctness_reward_func": 1.8803124763071537,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8580933725461364,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.845620596781373,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.2312187552452087,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.2053750045597553,
       "step": 16
     },
     {
-      "completion_length": 193.28125,
+      "completion_length": 193.40625,
       "epoch": 0.9,
-      "grad_norm": 82.87174987792969,
-      "kl": 52009.73198760301,
+      "grad_norm": 242775824.0,
+      "kl": 4245009.956890487,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 52.0097,
-      "reward": 6.219917170703411,
-      "reward_std": 0.7379699356388301,
-      "rewards/concensus_correctness_reward_func": 1.7056249976158142,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 4245.0103,
+      "reward": 6.807981312274933,
+      "reward_std": 0.4588104040012695,
+      "rewards/concensus_correctness_reward_func": 2.1166875064373016,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9275734014809132,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.864137526601553,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.180468749254942,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.1865312606096268,
       "step": 18
     },
     {
-      "completion_length": 194.3125,
+      "completion_length": 203.71875,
       "epoch": 1.0,
-      "grad_norm": 25.234548568725586,
-      "kl": 0.5797247909940779,
+      "grad_norm": 112.28648376464844,
+      "kl": 1089.226310593076,
       "learning_rate": 0.0,
-      "loss": 0.0006,
-      "reward": 6.682922422885895,
-      "reward_std": 0.429753286880441,
-      "rewards/concensus_correctness_reward_func": 2.0764999873936176,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 1.0892,
+      "reward": 6.604493856430054,
+      "reward_std": 0.6602307197172195,
+      "rewards/concensus_correctness_reward_func": 1.9911249987781048,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.9313287772238255,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.9178376197814941,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.20634376257658,
+      "rewards/strict_format_reward_func": 0.296875,
+      "rewards/xmlcount_reward_func": 1.2111562639474869,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 818.6132557076751,
-      "train_runtime": 193.9967,
-      "train_samples_per_second": 1.65,
+      "train_loss": 38319.64781138357,
+      "train_runtime": 193.4127,
+      "train_samples_per_second": 1.654,
       "train_steps_per_second": 0.103
     }
   ],