End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.727621313169948e-06,
-    "train_runtime": 377.7644,
-    "train_samples": 160,
-    "train_samples_per_second": 0.847,
-    "train_steps_per_second": 0.053
+    "train_loss": 1.7470321199652972e-05,
+    "train_runtime": 777.5233,
+    "train_samples": 316,
+    "train_samples_per_second": 0.412,
+    "train_steps_per_second": 0.026
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b98f651468c2d65adf2a8906702d308cef3335895e599b86b8b670cb4be53ca0
+oid sha256:8c98923f941bb1c89087d2d86c413aa24697fbf13481a19214e276a473ca53cc
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.727621313169948e-06,
-    "train_runtime": 377.7644,
-    "train_samples": 160,
-    "train_samples_per_second": 0.847,
-    "train_steps_per_second": 0.053
+    "train_loss": 1.7470321199652972e-05,
+    "train_runtime": 777.5233,
+    "train_samples": 316,
+    "train_samples_per_second": 0.412,
+    "train_steps_per_second": 0.026
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 0.5063291139240507,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 405.0,
-      "epoch": 0.1,
-      "grad_norm": 3.542961359024048,
+      "completion_length": 390.625,
+      "epoch": 0.05063291139240506,
+      "grad_norm": 19.693897247314453,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": 0.0,
-      "reward": 1.4710291721858084,
-      "reward_std": 1.442350228317082,
-      "rewards/concensus_correctness_reward_func": 0.6257500000065193,
+      "loss": -0.0,
+      "reward": 0.5654189372435212,
+      "reward_std": 0.6884031481022248,
+      "rewards/concensus_correctness_reward_func": 0.10256250202655792,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.41421666427049786,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.10379392973845825,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.243562500923872,
+      "rewards/xmlcount_reward_func": 0.10906249936670065,
       "step": 2
     },
     {
-      "completion_length": 457.65625,
-      "epoch": 0.2,
-      "grad_norm": 3.708958625793457,
-      "kl": 0.0009309585911978502,
+      "completion_length": 423.5,
+      "epoch": 0.10126582278481013,
+      "grad_norm": 15.050867080688477,
+      "kl": 0.0015515557388425805,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0,
-      "reward": 1.0695764030097052,
-      "reward_std": 0.8023686404339969,
-      "rewards/concensus_correctness_reward_func": 0.08512499928474426,
-      "rewards/consensus_reward_func": 0.125,
+      "loss": -0.0,
+      "reward": 0.41121767554432154,
+      "reward_std": 0.2975026599524426,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5073889201739803,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.10174894565716386,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.35206249984912574,
+      "rewards/xmlcount_reward_func": -0.003031250322237611,
       "step": 4
     },
     {
-      "completion_length": 452.5625,
-      "epoch": 0.3,
-      "grad_norm": 4.777602672576904,
-      "kl": 0.0013669177387782838,
+      "completion_length": 302.0625,
+      "epoch": 0.1518987341772152,
+      "grad_norm": 105.33124542236328,
+      "kl": 0.01090021853451617,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 1.1037784151558299,
-      "reward_std": 0.9202575778181199,
-      "rewards/concensus_correctness_reward_func": 0.12250000238418579,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.016203501319978386,
+      "reward_std": 0.8818220303510316,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.4263096438080538,
+      "rewards/question_recreation_reward_func": 0.10098475415725261,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11746875196695328,
+      "rewards/xmlcount_reward_func": -0.33478125697001815,
       "step": 6
     },
     {
-      "completion_length": 460.96875,
-      "epoch": 0.4,
-      "grad_norm": 4.447817325592041,
-      "kl": 0.0016320224494847935,
+      "completion_length": 376.53125,
+      "epoch": 0.20253164556962025,
+      "grad_norm": 6.963199138641357,
+      "kl": 0.00975047947395069,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 0.974716882687062,
-      "reward_std": 1.3214201303198934,
-      "rewards/concensus_correctness_reward_func": 0.13868750259280205,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 2.377215757209342,
+      "reward_std": 1.1443396444774407,
+      "rewards/concensus_correctness_reward_func": 1.875,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.46731058473233134,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.15671575430314988,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.10309376195073128,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": -0.029499998316168785,
       "step": 8
     },
     {
-      "completion_length": 388.28125,
-      "epoch": 0.5,
-      "grad_norm": 5.731485366821289,
-      "kl": 0.007084455588483252,
+      "completion_length": 326.84375,
+      "epoch": 0.25316455696202533,
+      "grad_norm": 13.946148872375488,
+      "kl": 0.0013114591529301833,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 0.7409542207606137,
-      "reward_std": 0.8507989638310391,
-      "rewards/concensus_correctness_reward_func": 0.018812499940395355,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.4975748333381489,
+      "reward_std": 0.4967209550086409,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4011729843914509,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.24107483273837715,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13346874713897705,
+      "rewards/xmlcount_reward_func": 0.13149999920278788,
       "step": 10
     },
     {
-      "completion_length": 452.84375,
-      "epoch": 0.6,
-      "grad_norm": 4.6150994300842285,
-      "kl": 0.004119118915696163,
+      "completion_length": 295.90625,
+      "epoch": 0.3037974683544304,
+      "grad_norm": 16.219940185546875,
+      "kl": 0.0020969353536202107,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 1.4759507272392511,
-      "reward_std": 1.8463579853996634,
-      "rewards/concensus_correctness_reward_func": 0.6328125,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.0173564599826932,
+      "reward_std": 1.5090986860057,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3737632445991039,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.19388769299257547,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.34437499660998583,
+      "rewards/xmlcount_reward_func": -0.11403124465141445,
       "step": 12
     },
     {
-      "completion_length": 445.71875,
-      "epoch": 0.7,
-      "grad_norm": 4.404483318328857,
-      "kl": 0.0020462702559598256,
+      "completion_length": 333.96875,
+      "epoch": 0.35443037974683544,
+      "grad_norm": 55.26883316040039,
+      "kl": 0.02062846499393345,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 0.8386208694428205,
-      "reward_std": 0.7303225318901241,
+      "reward": 0.4555127089843154,
+      "reward_std": 0.49959600171496277,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4480896024033427,
+      "rewards/question_recreation_reward_func": 0.18432520679198205,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3280312493443489,
+      "rewards/xmlcount_reward_func": 0.14618749916553497,
       "step": 14
     },
     {
-      "completion_length": 441.46875,
-      "epoch": 0.8,
-      "grad_norm": 3.878316879272461,
-      "kl": 0.002502717266906984,
+      "completion_length": 350.71875,
+      "epoch": 0.4050632911392405,
+      "grad_norm": 131.1157684326172,
+      "kl": 0.11864930617957725,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0,
-      "reward": 1.5705258045345545,
-      "reward_std": 0.7504621744155884,
-      "rewards/concensus_correctness_reward_func": 0.04600000008940697,
-      "rewards/consensus_reward_func": 0.1875,
+      "loss": 0.0001,
+      "reward": 0.45018581731710583,
+      "reward_std": 0.4574645821703598,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5816507926210761,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.13740457454696298,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5678749927319586,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.10965625010430813,
       "step": 16
     },
     {
-      "completion_length": 429.53125,
-      "epoch": 0.9,
-      "grad_norm": 4.667481899261475,
-      "kl": 0.0050733851312543266,
+      "completion_length": 311.78125,
+      "epoch": 0.45569620253164556,
+      "grad_norm": 28.433929443359375,
+      "kl": 0.002976935877086362,
       "learning_rate": 1.3545689574841341e-08,
       "loss": 0.0,
-      "reward": 1.3714904030784965,
-      "reward_std": 0.9252548192162067,
-      "rewards/concensus_correctness_reward_func": 0.20068750344216824,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.34964469261467457,
+      "reward_std": 0.34213452675612643,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.40092792897485197,
+      "rewards/question_recreation_reward_func": 0.23998843505978584,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5198749983683228,
+      "rewards/xmlcount_reward_func": 0.04715625708922744,
       "step": 18
     },
     {
-      "completion_length": 426.34375,
-      "epoch": 1.0,
-      "grad_norm": 4.335679054260254,
-      "kl": 0.002546306644944707,
+      "completion_length": 227.90625,
+      "epoch": 0.5063291139240507,
+      "grad_norm": 9.338957786560059,
+      "kl": 0.013950712120276876,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 0.981014184653759,
-      "reward_std": 0.4036117763584457,
-      "rewards/concensus_correctness_reward_func": 0.06537499837577343,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.5095127180684358,
+      "reward_std": 0.5383867279160768,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4608266893774271,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.13626271882094443,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2673124959692359,
+      "rewards/xmlcount_reward_func": 0.12324999878183007,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 0.5063291139240507,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.727621313169948e-06,
-      "train_runtime": 377.7644,
-      "train_samples_per_second": 0.847,
-      "train_steps_per_second": 0.053
+      "train_loss": 1.7470321199652972e-05,
+      "train_runtime": 777.5233,
+      "train_samples_per_second": 0.412,
+      "train_steps_per_second": 0.026
     }
   ],
   "logging_steps": 2,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:577f25fa8e3e784bc7ee5e169a012f7020e2d21576732b30eeaf080e28b4ed43
+oid sha256:ce77f6e14977aab39201ed9d80237953c712a0476f7b2159056c099d7befbfa7
 size 6008