End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 38319.64781138357,
-    "train_runtime": 193.4127,
+    "train_loss": 1713.5320490076206,
+    "train_runtime": 185.6881,
     "train_samples": 160,
-    "train_samples_per_second": 1.654,
-    "train_steps_per_second": 0.103
+    "train_samples_per_second": 1.723,
+    "train_steps_per_second": 0.108
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7dbe3f92d3e8e56ea6129db6c3d433579cc4895b85e9fccc3e679ac4f6206ed0
+oid sha256:f15cdd9ca73aba6fe0a6fbd3cfcbcffd646a37d963a3c4d3ea130e46bc3e86e5
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 38319.64781138357,
-    "train_runtime": 193.4127,
+    "train_loss": 1713.5320490076206,
+    "train_runtime": 185.6881,
     "train_samples": 160,
-    "train_samples_per_second": 1.654,
-    "train_steps_per_second": 0.103
+    "train_samples_per_second": 1.723,
+    "train_steps_per_second": 0.108
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 268.125,
+      "completion_length": 221.125,
       "epoch": 0.1,
-      "grad_norm": 52.51573181152344,
+      "grad_norm": 20.139453887939453,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 3.7880874909460545,
-      "reward_std": 0.7253511965391226,
-      "rewards/concensus_correctness_reward_func": 0.8900625109672546,
-      "rewards/consensus_reward_func": 1.0625,
+      "reward": 3.583044346421957,
+      "reward_std": 0.3225670934189111,
+      "rewards/concensus_correctness_reward_func": 0.8732500001788139,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5797749441117048,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6593567873351276,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 0.9120000023394823,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 0.8629375072196126,
       "step": 2
     },
     {
-      "completion_length": 207.125,
+      "completion_length": 218.1875,
       "epoch": 0.2,
-      "grad_norm": 14097.90234375,
-      "kl": 530.8586953068152,
+      "grad_norm": 9141.5224609375,
+      "kl": 204.94856747821905,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.5309,
-      "reward": 5.9291402995586395,
-      "reward_std": 0.2267392118810676,
-      "rewards/concensus_correctness_reward_func": 1.6063750013709068,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.2049,
+      "reward": 5.809427872300148,
+      "reward_std": 0.40486387093551457,
+      "rewards/concensus_correctness_reward_func": 1.5327500030398369,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9300465881824493,
+      "rewards/question_recreation_reward_func": 0.9273653849959373,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.1739687621593475,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.1618125215172768,
       "step": 4
     },
     {
-      "completion_length": 213.8125,
+      "completion_length": 236.0,
       "epoch": 0.3,
-      "grad_norm": 1793.24462890625,
-      "kl": 1615.383490987122,
+      "grad_norm": 77.85133361816406,
+      "kl": 1335.9335167701356,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 1.6154,
-      "reward": 6.197655111551285,
-      "reward_std": 0.5888944863327197,
-      "rewards/concensus_correctness_reward_func": 1.7123124990612268,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 1.3359,
+      "reward": 5.69304946064949,
+      "reward_std": 0.8221923279052135,
+      "rewards/concensus_correctness_reward_func": 1.5556250112131238,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9221238419413567,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.887486869469285,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.2038437500596046,
+      "rewards/xmlcount_reward_func": 1.140562504529953,
       "step": 6
     },
     {
-      "completion_length": 237.3125,
+      "completion_length": 197.375,
       "epoch": 0.4,
-      "grad_norm": 2184983.5,
-      "kl": 80967.61272667907,
+      "grad_norm": 377.45550537109375,
+      "kl": 50.22693386673927,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 80.9676,
-      "reward": 6.5779251754283905,
-      "reward_std": 0.5500927083194256,
-      "rewards/concensus_correctness_reward_func": 1.9547500163316727,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0502,
+      "reward": 6.55130061507225,
+      "reward_std": 0.3447979334741831,
+      "rewards/concensus_correctness_reward_func": 2.0681250169873238,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.9303939752280712,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.8836442492902279,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.1927812546491623,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 1.1620312705636024,
       "step": 8
     },
     {
-      "completion_length": 184.125,
+      "completion_length": 165.53125,
       "epoch": 0.5,
-      "grad_norm": 28116684800.0,
-      "kl": 378830514.4991078,
+      "grad_norm": 1032759616.0,
+      "kl": 16418408.761649942,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 378830.5938,
-      "reward": 6.973221987485886,
-      "reward_std": 0.4501988561823964,
-      "rewards/concensus_correctness_reward_func": 2.1776249930262566,
+      "loss": 16418.4082,
+      "reward": 6.737318724393845,
+      "reward_std": 0.39243057603016496,
+      "rewards/concensus_correctness_reward_func": 2.025687500834465,
       "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.8454405581578612,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.9018187988549471,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1845312640070915,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.2160625085234642,
       "step": 10
     },
     {
-      "completion_length": 220.78125,
+      "completion_length": 183.96875,
       "epoch": 0.6,
-      "grad_norm": 538161.25,
-      "kl": 36245.77570069302,
+      "grad_norm": 69.0755615234375,
+      "kl": 16.420032211579382,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 36.2458,
-      "reward": 6.162264287471771,
-      "reward_std": 0.28252823650836945,
-      "rewards/concensus_correctness_reward_func": 1.7267500013113022,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0164,
+      "reward": 6.181130200624466,
+      "reward_std": 0.4472991875372827,
+      "rewards/concensus_correctness_reward_func": 1.6567499972879887,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8918579369783401,
+      "rewards/question_recreation_reward_func": 0.9348177053034306,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.2311562672257423,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.2458125054836273,
       "step": 12
     },
     {
-      "completion_length": 202.78125,
+      "completion_length": 184.96875,
       "epoch": 0.7,
-      "grad_norm": 1045.7855224609375,
-      "kl": 208.7559353262186,
+      "grad_norm": 10738374.0,
+      "kl": 715220.8279513596,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.2088,
-      "reward": 6.0759420692920685,
-      "reward_std": 0.5171458608820103,
-      "rewards/concensus_correctness_reward_func": 1.8015624918043613,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 715.2209,
+      "reward": 6.46547394990921,
+      "reward_std": 0.6154752657748759,
+      "rewards/concensus_correctness_reward_func": 1.9638124890625477,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.8865983616560698,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.8580052405595779,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 1.1690312623977661,
+      "rewards/xmlcount_reward_func": 1.1749062612652779,
       "step": 14
     },
     {
-      "completion_length": 200.25,
+      "completion_length": 183.25,
       "epoch": 0.8,
-      "grad_norm": 108.79785919189453,
-      "kl": 216.4876526929438,
+      "grad_norm": 66.06885528564453,
+      "kl": 42.09273688448593,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.2165,
-      "reward": 6.525058060884476,
-      "reward_std": 0.9049128945916891,
-      "rewards/concensus_correctness_reward_func": 1.8803124763071537,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0421,
+      "reward": 6.479139924049377,
+      "reward_std": 0.39258914068341255,
+      "rewards/concensus_correctness_reward_func": 1.8927499912679195,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.845620596781373,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.7949523320421576,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.2053750045597553,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.1976875104010105,
       "step": 16
     },
     {
-      "completion_length": 193.40625,
+      "completion_length": 171.3125,
       "epoch": 0.9,
-      "grad_norm": 242775824.0,
-      "kl": 4245009.956890487,
+      "grad_norm": 223.96368408203125,
+      "kl": 12.877988177351654,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 4245.0103,
-      "reward": 6.807981312274933,
-      "reward_std": 0.4588104040012695,
-      "rewards/concensus_correctness_reward_func": 2.1166875064373016,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0129,
+      "reward": 6.037820547819138,
+      "reward_std": 0.5467852650908753,
+      "rewards/concensus_correctness_reward_func": 1.866249993443489,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.864137526601553,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.7587268259376287,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1865312606096268,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.162843756377697,
       "step": 18
     },
     {
-      "completion_length": 203.71875,
+      "completion_length": 187.625,
       "epoch": 1.0,
-      "grad_norm": 112.28648376464844,
-      "kl": 1089.226310593076,
+      "grad_norm": 610.2742309570312,
+      "kl": 28.89718913845718,
       "learning_rate": 0.0,
-      "loss": 1.0892,
-      "reward": 6.604493856430054,
-      "reward_std": 0.6602307197172195,
-      "rewards/concensus_correctness_reward_func": 1.9911249987781048,
+      "loss": 0.0289,
+      "reward": 6.34273362159729,
+      "reward_std": 0.3181899096816778,
+      "rewards/concensus_correctness_reward_func": 1.8753749877214432,
       "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.9178376197814941,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9278585948050022,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.2111562639474869,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.1801250167191029,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 38319.64781138357,
-      "train_runtime": 193.4127,
-      "train_samples_per_second": 1.654,
-      "train_steps_per_second": 0.103
+      "train_loss": 1713.5320490076206,
+      "train_runtime": 185.6881,
+      "train_samples_per_second": 1.723,
+      "train_steps_per_second": 0.108
     }
   ],
   "logging_steps": 2,