End of training

README.md

@@ -8,6 +8,7 @@ tags:
 - grpo
 - gensyn
 - I am wary playful sandpiper
+- unsloth
 - trl
 licence: license
 ---
@@ -38,8 +39,8 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.51.1
-- Pytorch: 2.5.1
+- Transformers: 4.50.3
+- Pytorch: 2.6.0
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1
 

adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "Gensyn/Qwen2.5-0.5B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_bias": false,
+  "lora_dropout": 0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "v_proj",
+    "up_proj",
+    "gate_proj",
+    "k_proj",
+    "down_proj",
+    "o_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2fd866e7529479db8185fe0ea87e8fbcc76f81fd80814dc272397f2b92e470a
+size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2722467361.9045286,
-    "train_runtime": 164.4325,
-    "train_samples": 28,
-    "train_samples_per_second": 1.946,
-    "train_steps_per_second": 0.122
+    "train_loss": 2.7457240321382415e-06,
+    "train_runtime": 1511.9519,
+    "train_samples": 11,
+    "train_samples_per_second": 0.212,
+    "train_steps_per_second": 0.013
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2722467361.9045286,
-    "train_runtime": 164.4325,
-    "train_samples": 28,
-    "train_samples_per_second": 1.946,
-    "train_steps_per_second": 0.122
+    "train_loss": 2.7457240321382415e-06,
+    "train_runtime": 1511.9519,
+    "train_samples": 11,
+    "train_samples_per_second": 0.212,
+    "train_steps_per_second": 0.013
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 6.7272727272727275,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 260.3125,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 75.30554962158203,
-      "kl": 0.0,
-      "learning_rate": 5e-07,
+      "completion_length": 402.5625,
+      "epoch": 0.7272727272727273,
+      "grad_norm": 2.2664785385131836,
+      "kl": 0.002372696340898983,
+      "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 3.717131495475769,
-      "reward_std": 1.6642041678423993,
-      "rewards/concensus_correctness_reward_func": 1.2423124946653843,
-      "rewards/consensus_reward_func": 0.875,
+      "reward": 0.40154129709117115,
+      "reward_std": 0.87543322192505,
+      "rewards/concensus_correctness_reward_func": 0.05193749815225601,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6729752793908119,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.23847879050299525,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8643437549471855,
+      "rewards/xmlcount_reward_func": 0.03300000017043203,
       "step": 2
     },
     {
-      "completion_length": 201.875,
-      "epoch": 1.0,
-      "grad_norm": 21.483325958251953,
-      "kl": 0.07919525541365147,
-      "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0001,
-      "reward": 4.991062005360921,
-      "reward_std": 0.5951124601997435,
-      "rewards/concensus_correctness_reward_func": 1.3690833350022633,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "completion_length": 527.4285714285714,
+      "epoch": 1.3636363636363638,
+      "grad_norm": 4.900257587432861,
+      "kl": 0.0018949694682045706,
+      "learning_rate": 4.698684378016222e-07,
+      "loss": 0.0,
+      "reward": 0.13510957573141372,
+      "reward_std": 0.4241678650890078,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8097285479307175,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.11714529685143914,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1455833117167156,
+      "rewards/xmlcount_reward_func": 0.01796428220612662,
       "step": 4
     },
     {
-      "completion_length": 179.75,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 1183044580933632.0,
-      "kl": 27216760719756.137,
-      "learning_rate": 4.472851273490984e-07,
-      "loss": 27216762880.0,
-      "reward": 4.413312576711178,
-      "reward_std": 1.359453333192505,
-      "rewards/concensus_correctness_reward_func": 1.1898749992251396,
-      "rewards/consensus_reward_func": 1.125,
+      "completion_length": 377.39285714285717,
+      "epoch": 2.0,
+      "grad_norm": 58.00647735595703,
+      "kl": 0.010912998637650162,
+      "learning_rate": 4.193203929064353e-07,
+      "loss": 0.0,
+      "reward": 0.4419659463954823,
+      "reward_std": 0.4245131053030491,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7928437702357769,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.21628737729042768,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1180937439203262,
+      "rewards/xmlcount_reward_func": 0.2256785716329302,
       "step": 6
     },
     {
-      "completion_length": 184.5,
-      "epoch": 2.0,
-      "grad_norm": 41.23329544067383,
-      "kl": 3.6586173151930175,
-      "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0027,
-      "reward": 4.694513579209645,
-      "reward_std": 0.35443124012090266,
-      "rewards/concensus_correctness_reward_func": 1.2385833263397217,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "completion_length": 417.59375,
+      "epoch": 2.7272727272727275,
+      "grad_norm": 28.17329978942871,
+      "kl": 0.002533846374717541,
+      "learning_rate": 3.5042385616324236e-07,
+      "loss": 0.0,
+      "reward": 0.3962969146668911,
+      "reward_std": 0.43166567012667656,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.8170136710007986,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2620781655423343,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1389166812102,
+      "rewards/xmlcount_reward_func": 0.13421875424683094,
       "step": 8
     },
     {
-      "completion_length": 199.34375,
-      "epoch": 2.571428571428571,
-      "grad_norm": 3330.87548828125,
-      "kl": 7910739151.801059,
-      "learning_rate": 3.1137137178519977e-07,
-      "loss": 7910739.0,
-      "reward": 4.204887181520462,
-      "reward_std": 1.4344781459658407,
-      "rewards/concensus_correctness_reward_func": 1.1611874923110008,
-      "rewards/consensus_reward_func": 1.125,
+      "completion_length": 499.0357142857143,
+      "epoch": 3.3636363636363638,
+      "grad_norm": 1.670122504234314,
+      "kl": 0.0034035631023081286,
+      "learning_rate": 2.706448363680831e-07,
+      "loss": 0.0,
+      "reward": 0.2196260052067893,
+      "reward_std": 0.3056698822017227,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.808262163773179,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.11680457075791699,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0479374751448631,
+      "rewards/xmlcount_reward_func": 0.1028214277965682,
       "step": 10
     },
     {
-      "completion_length": 187.33333333333334,
-      "epoch": 3.0,
-      "grad_norm": 1041.962890625,
-      "kl": 32.96611806191504,
-      "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0247,
-      "reward": 4.72384387254715,
-      "reward_std": 0.6615689925492916,
-      "rewards/concensus_correctness_reward_func": 1.287333329518636,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "completion_length": 306.60714285714283,
+      "epoch": 4.0,
+      "grad_norm": 96.93277740478516,
+      "kl": 0.001341122152682926,
+      "learning_rate": 1.886286282148002e-07,
+      "loss": 0.0,
+      "reward": 0.22444776924593107,
+      "reward_std": 0.23045456409454346,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.7938854570190111,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.14151919420276368,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0592916409174602,
+      "rewards/xmlcount_reward_func": 0.08292856865695544,
       "step": 12
     },
     {
-      "completion_length": 187.375,
-      "epoch": 3.571428571428571,
-      "grad_norm": 65.9546890258789,
-      "kl": 3.378439131192863,
-      "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0034,
-      "reward": 5.0007642805576324,
-      "reward_std": 1.2876124810427427,
-      "rewards/concensus_correctness_reward_func": 1.456375002861023,
-      "rewards/consensus_reward_func": 1.3125,
+      "completion_length": 356.375,
+      "epoch": 4.7272727272727275,
+      "grad_norm": 1.8176085948944092,
+      "kl": 0.0011140671704197302,
+      "learning_rate": 1.1326296046939333e-07,
+      "loss": 0.0,
+      "reward": 0.20134539902210236,
+      "reward_std": 0.31388112530112267,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.823857918381691,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.19462664891034365,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.158031228929758,
+      "rewards/xmlcount_reward_func": 0.006718748714774847,
       "step": 14
     },
     {
-      "completion_length": 183.79166666666666,
-      "epoch": 4.0,
-      "grad_norm": 25.8631534576416,
-      "kl": 2.2448727103571096,
-      "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0017,
-      "reward": 3.6760837137699127,
-      "reward_std": 0.7920290327941378,
-      "rewards/concensus_correctness_reward_func": 0.8623333324988683,
-      "rewards/consensus_reward_func": 0.8333333333333334,
+      "completion_length": 318.5357142857143,
+      "epoch": 5.363636363636363,
+      "grad_norm": 1.7451318502426147,
+      "kl": 0.00214366678847,
+      "learning_rate": 5.271487265090163e-08,
+      "loss": 0.0,
+      "reward": 0.2694588005542755,
+      "reward_std": 0.2729693810854639,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.830000251531601,
+      "rewards/question_recreation_reward_func": 0.1898516629423414,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.150416652361552,
+      "rewards/xmlcount_reward_func": 0.07960714266768523,
       "step": 16
     },
     {
-      "completion_length": 194.65625,
-      "epoch": 4.571428571428571,
-      "grad_norm": 70.06986236572266,
-      "kl": 11.929521653335541,
-      "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0119,
-      "reward": 5.072862520813942,
-      "reward_std": 0.4147999930282822,
-      "rewards/concensus_correctness_reward_func": 1.3826249986886978,
-      "rewards/consensus_reward_func": 1.5625,
+      "completion_length": 381.17857142857144,
+      "epoch": 6.0,
+      "grad_norm": 2.0452146530151367,
+      "kl": 0.002669754546202187,
+      "learning_rate": 1.3545689574841341e-08,
+      "loss": 0.0,
+      "reward": 0.33125837094017435,
+      "reward_std": 0.18709695139633759,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8551437705755234,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.24400837253779173,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.131968729197979,
+      "rewards/xmlcount_reward_func": 0.08724999481013843,
       "step": 18
     },
     {
-      "completion_length": 185.0,
-      "epoch": 5.0,
-      "grad_norm": 101.4698486328125,
-      "kl": 1.009298223691682,
-      "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0008,
-      "reward": 4.266951421896617,
-      "reward_std": 1.550357508783539,
-      "rewards/concensus_correctness_reward_func": 1.181999996304512,
-      "rewards/consensus_reward_func": 1.0833333333333333,
+      "completion_length": 354.5625,
+      "epoch": 6.7272727272727275,
+      "grad_norm": 1.446305513381958,
+      "kl": 0.001905521028675139,
+      "learning_rate": 0.0,
+      "loss": 0.0,
+      "reward": 0.5082273744046688,
+      "reward_std": 0.49512116867117584,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.7240346843997637,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.28819612972438335,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.110916664203008,
+      "rewards/xmlcount_reward_func": 0.15753125306218863,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 6.7272727272727275,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2722467361.9045286,
-      "train_runtime": 164.4325,
-      "train_samples_per_second": 1.946,
-      "train_steps_per_second": 0.122
+      "train_loss": 2.7457240321382415e-06,
+      "train_runtime": 1511.9519,
+      "train_samples_per_second": 0.212,
+      "train_steps_per_second": 0.013
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 10,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -227,7 +227,7 @@
     }
   },
   "total_flos": 0.0,
-  "train_batch_size": 2,
+  "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c940b13e89e5826b7197d192e7ce1cb7c27a30f9db0b401fd0495f422c863410
-size 5944
+oid sha256:40da36acc51b0694a52552ca44416433ff9dc04afbe8c57af271ea91a2ff1d1a
+size 6072