End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 47.32144293799065,
-    "train_runtime": 162.7274,
+    "train_loss": 2722467361.9045286,
+    "train_runtime": 164.4325,
     "train_samples": 28,
-    "train_samples_per_second": 1.966,
-    "train_steps_per_second": 0.123
+    "train_samples_per_second": 1.946,
+    "train_steps_per_second": 0.122
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:522e8e5a75bfa7fe97e6ae3bc8bec9eed01b090c0544351cf179a16afba71a1a
+oid sha256:25dbb610256ab3aa0a97f054fa0da39e8b766816031f4dcab359bc391ae65724
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 47.32144293799065,
-    "train_runtime": 162.7274,
+    "train_loss": 2722467361.9045286,
+    "train_runtime": 164.4325,
     "train_samples": 28,
-    "train_samples_per_second": 1.966,
-    "train_steps_per_second": 0.123
+    "train_samples_per_second": 1.946,
+    "train_steps_per_second": 0.122
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 220.8125,
+      "completion_length": 260.3125,
       "epoch": 0.5714285714285714,
-      "grad_norm": 31.785186767578125,
+      "grad_norm": 75.30554962158203,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 2.792306764051318,
-      "reward_std": 0.5164533269235108,
-      "rewards/concensus_correctness_reward_func": 0.4905625022947788,
-      "rewards/consensus_reward_func": 0.625,
+      "loss": 0.0,
+      "reward": 3.717131495475769,
+      "reward_std": 1.6642041678423993,
+      "rewards/concensus_correctness_reward_func": 1.2423124946653843,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7258380672428757,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6729752793908119,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8259062492288649,
+      "rewards/xmlcount_reward_func": 0.8643437549471855,
       "step": 2
     },
     {
-      "completion_length": 206.875,
+      "completion_length": 201.875,
       "epoch": 1.0,
-      "grad_norm": 122.33345794677734,
-      "kl": 2.1089118261976787,
+      "grad_norm": 21.483325958251953,
+      "kl": 0.07919525541365147,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0016,
-      "reward": 5.921782890955607,
-      "reward_std": 0.693140882688264,
-      "rewards/concensus_correctness_reward_func": 2.2554166664679847,
-      "rewards/consensus_reward_func": 1.3333333333333333,
+      "loss": 0.0001,
+      "reward": 4.991062005360921,
+      "reward_std": 0.5951124601997435,
+      "rewards/concensus_correctness_reward_func": 1.3690833350022633,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6714495979249477,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8097285479307175,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.16158331433932,
+      "rewards/xmlcount_reward_func": 1.1455833117167156,
       "step": 4
     },
     {
-      "completion_length": 194.9375,
+      "completion_length": 179.75,
       "epoch": 1.5714285714285714,
-      "grad_norm": 95.088623046875,
-      "kl": 23.749349858495407,
+      "grad_norm": 1183044580933632.0,
+      "kl": 27216760719756.137,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0237,
-      "reward": 5.464593216776848,
-      "reward_std": 1.1199153698980808,
-      "rewards/concensus_correctness_reward_func": 1.8176249973475933,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 27216762880.0,
+      "reward": 4.413312576711178,
+      "reward_std": 1.359453333192505,
+      "rewards/concensus_correctness_reward_func": 1.1898749992251396,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.7988119628280401,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.7928437702357769,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1606562212109566,
+      "rewards/xmlcount_reward_func": 1.1180937439203262,
       "step": 6
     },
     {
-      "completion_length": 184.70833333333334,
+      "completion_length": 184.5,
       "epoch": 2.0,
-      "grad_norm": 776.6046142578125,
-      "kl": 622373.1227876631,
+      "grad_norm": 41.23329544067383,
+      "kl": 3.6586173151930175,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 466.7799,
-      "reward": 4.530350744724274,
-      "reward_std": 1.4232008809728238,
-      "rewards/concensus_correctness_reward_func": 1.124416669209798,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0027,
+      "reward": 4.694513579209645,
+      "reward_std": 0.35443124012090266,
+      "rewards/concensus_correctness_reward_func": 1.2385833263397217,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.8376424163579941,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.8170136710007986,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1516249626874924,
+      "rewards/xmlcount_reward_func": 1.1389166812102,
       "step": 8
     },
     {
-      "completion_length": 193.875,
+      "completion_length": 199.34375,
       "epoch": 2.571428571428571,
-      "grad_norm": 7172.392578125,
-      "kl": 285.61230134323705,
+      "grad_norm": 3330.87548828125,
+      "kl": 7910739151.801059,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.2856,
-      "reward": 4.841844812035561,
-      "reward_std": 1.6193378757598111,
-      "rewards/concensus_correctness_reward_func": 1.285937488079071,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": 7910739.0,
+      "reward": 4.204887181520462,
+      "reward_std": 1.4344781459658407,
+      "rewards/concensus_correctness_reward_func": 1.1611874923110008,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.8501260448247194,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.808262163773179,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 1.1276562362909317,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 1.0479374751448631,
       "step": 10
     },
     {
-      "completion_length": 186.5,
+      "completion_length": 187.33333333333334,
       "epoch": 3.0,
-      "grad_norm": 35.74233627319336,
-      "kl": 208.1803734153509,
+      "grad_norm": 1041.962890625,
+      "kl": 32.96611806191504,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.1561,
-      "reward": 5.420761525630951,
-      "reward_std": 0.8128547375478471,
-      "rewards/concensus_correctness_reward_func": 1.6499166786670685,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0247,
+      "reward": 4.72384387254715,
+      "reward_std": 0.6615689925492916,
+      "rewards/concensus_correctness_reward_func": 1.287333329518636,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8841781715552012,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.7938854570190111,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1366666704416275,
+      "rewards/xmlcount_reward_func": 1.0592916409174602,
       "step": 12
     },
     {
-      "completion_length": 195.5625,
+      "completion_length": 187.375,
       "epoch": 3.571428571428571,
-      "grad_norm": 87303.8671875,
-      "kl": 2730.737289617886,
+      "grad_norm": 65.9546890258789,
+      "kl": 3.378439131192863,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 2.7307,
-      "reward": 5.099680621176958,
-      "reward_std": 0.8015707243248471,
-      "rewards/concensus_correctness_reward_func": 1.429125003516674,
+      "loss": 0.0034,
+      "reward": 5.0007642805576324,
+      "reward_std": 1.2876124810427427,
+      "rewards/concensus_correctness_reward_func": 1.456375002861023,
       "rewards/consensus_reward_func": 1.3125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.795399272814393,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.823857918381691,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1251562237739563,
+      "rewards/xmlcount_reward_func": 1.158031228929758,
       "step": 14
     },
     {
-      "completion_length": 189.66666666666666,
+      "completion_length": 183.79166666666666,
       "epoch": 4.0,
-      "grad_norm": 123.0419692993164,
-      "kl": 9.184569627046585,
+      "grad_norm": 25.8631534576416,
+      "kl": 2.2448727103571096,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0069,
-      "reward": 4.068873226642609,
-      "reward_std": 1.8017581754053633,
-      "rewards/concensus_correctness_reward_func": 1.0760833323001862,
+      "loss": 0.0017,
+      "reward": 3.6760837137699127,
+      "reward_std": 0.7920290327941378,
+      "rewards/concensus_correctness_reward_func": 0.8623333324988683,
       "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.852789948383967,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.830000251531601,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0566666399439175,
+      "rewards/xmlcount_reward_func": 1.150416652361552,
       "step": 16
     },
     {
-      "completion_length": 178.6875,
+      "completion_length": 194.65625,
       "epoch": 4.571428571428571,
-      "grad_norm": 156.2207794189453,
-      "kl": 3191.5056470287964,
+      "grad_norm": 70.06986236572266,
+      "kl": 11.929521653335541,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 3.1915,
-      "reward": 5.771281331777573,
-      "reward_std": 1.3582927275274415,
-      "rewards/concensus_correctness_reward_func": 1.7130624987185001,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0119,
+      "reward": 5.072862520813942,
+      "reward_std": 0.4147999930282822,
+      "rewards/concensus_correctness_reward_func": 1.3826249986886978,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5625,
-      "rewards/question_recreation_reward_func": 0.8282500533387065,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8551437705755234,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1674687415361404,
+      "rewards/xmlcount_reward_func": 1.131968729197979,
       "step": 18
     },
     {
-      "completion_length": 194.08333333333334,
+      "completion_length": 185.0,
       "epoch": 5.0,
-      "grad_norm": 1508.404296875,
-      "kl": 51.10552144042837,
+      "grad_norm": 101.4698486328125,
+      "kl": 1.009298223691682,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0383,
-      "reward": 4.8991721868515015,
-      "reward_std": 0.9636430108221248,
-      "rewards/concensus_correctness_reward_func": 1.5873333364725113,
+      "loss": 0.0008,
+      "reward": 4.266951421896617,
+      "reward_std": 1.550357508783539,
+      "rewards/concensus_correctness_reward_func": 1.181999996304512,
       "rewards/consensus_reward_func": 1.0833333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8430471594134966,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.7240346843997637,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1354583154122035,
+      "rewards/xmlcount_reward_func": 1.110916664203008,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 47.32144293799065,
-      "train_runtime": 162.7274,
-      "train_samples_per_second": 1.966,
-      "train_steps_per_second": 0.123
+      "train_loss": 2722467361.9045286,
+      "train_runtime": 164.4325,
+      "train_samples_per_second": 1.946,
+      "train_steps_per_second": 0.122
     }
   ],
   "logging_steps": 2,