Model save

README.md

@@ -0,0 +1,68 @@
+---
+base_model: Qwen/Qwen2.5-VL-3B-Instruct
+library_name: transformers
+model_name: Qwen2.5-VL-3B-Instruct-Open-R1-Distill-select
+tags:
+- generated_from_trainer
+- trl
+- grpo
+licence: license
+---
+
+# Model Card for Qwen2.5-VL-3B-Instruct-Open-R1-Distill-select
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-VL-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="zzhang1987/Qwen2.5-VL-3B-Instruct-Open-R1-Distill-select", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/causalai/huggingface/runs/e1ladt2z) 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.15.0.dev0
+- Transformers: 4.49.0.dev0
+- Pytorch: 2.5.1
+- Datasets: 3.2.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.08425404627709404,
+    "train_runtime": 290681.8756,
+    "train_samples": 17056,
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.0
+}

generation_config.json

@@ -0,0 +1,16 @@
+{
+  "attn_implementation": "flash_attention_2",
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.1,
+  "top_k": 1,
+  "top_p": 0.001,
+  "transformers_version": "4.49.0.dev0",
+  "use_cache": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.08425404627709404,
+    "train_runtime": 290681.8756,
+    "train_samples": 17056,
+    "train_samples_per_second": 0.059,
+    "train_steps_per_second": 0.0
+}

trainer_state.json

@@ -0,0 +1,400 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.99812382739212,
+  "eval_steps": 100,
+  "global_step": 133,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 83.688671875,
+      "epoch": 0.0375234521575985,
+      "grad_norm": 2.9919118881225586,
+      "kl": 0.01474391222000122,
+      "learning_rate": 7.1428571428571436e-06,
+      "loss": 0.0006,
+      "reward": 0.064453125,
+      "reward_std": 0.10932583417743444,
+      "rewards/accuracy_reward": 0.03828125,
+      "rewards/format_reward": 0.026171875,
+      "step": 5
+    },
+    {
+      "completion_length": 67.101953125,
+      "epoch": 0.075046904315197,
+      "grad_norm": 6.934049129486084,
+      "kl": 0.3821044921875,
+      "learning_rate": 1.4285714285714287e-05,
+      "loss": 0.0153,
+      "reward": 1.072265625,
+      "reward_std": 0.37313607819378375,
+      "rewards/accuracy_reward": 0.280078125,
+      "rewards/format_reward": 0.7921875,
+      "step": 10
+    },
+    {
+      "completion_length": 67.77265625,
+      "epoch": 0.1125703564727955,
+      "grad_norm": 4.7614545822143555,
+      "kl": 0.9060546875,
+      "learning_rate": 1.9996515418688493e-05,
+      "loss": 0.0363,
+      "reward": 1.28203125,
+      "reward_std": 0.3224642466753721,
+      "rewards/accuracy_reward": 0.357421875,
+      "rewards/format_reward": 0.924609375,
+      "step": 15
+    },
+    {
+      "completion_length": 46.908984375,
+      "epoch": 0.150093808630394,
+      "grad_norm": 2.170837640762329,
+      "kl": 1.436572265625,
+      "learning_rate": 1.9874809871741877e-05,
+      "loss": 0.0574,
+      "reward": 1.243359375,
+      "reward_std": 0.29788401871919634,
+      "rewards/accuracy_reward": 0.354296875,
+      "rewards/format_reward": 0.8890625,
+      "step": 20
+    },
+    {
+      "completion_length": 19.840625,
+      "epoch": 0.18761726078799248,
+      "grad_norm": 11.829867362976074,
+      "kl": 1.926123046875,
+      "learning_rate": 1.9581296124106682e-05,
+      "loss": 0.077,
+      "reward": 1.378515625,
+      "reward_std": 0.1831501940265298,
+      "rewards/accuracy_reward": 0.394921875,
+      "rewards/format_reward": 0.98359375,
+      "step": 25
+    },
+    {
+      "completion_length": 23.02890625,
+      "epoch": 0.225140712945591,
+      "grad_norm": 2.8077869415283203,
+      "kl": 1.7828125,
+      "learning_rate": 1.912108091398988e-05,
+      "loss": 0.0713,
+      "reward": 1.265234375,
+      "reward_std": 0.2775565652176738,
+      "rewards/accuracy_reward": 0.3453125,
+      "rewards/format_reward": 0.919921875,
+      "step": 30
+    },
+    {
+      "completion_length": 143.800390625,
+      "epoch": 0.2626641651031895,
+      "grad_norm": 4.427940845489502,
+      "kl": 5.769921875,
+      "learning_rate": 1.8502171357296144e-05,
+      "loss": 0.2308,
+      "reward": 1.275,
+      "reward_std": 0.3242990938946605,
+      "rewards/accuracy_reward": 0.38828125,
+      "rewards/format_reward": 0.88671875,
+      "step": 35
+    },
+    {
+      "completion_length": 78.410546875,
+      "epoch": 0.300187617260788,
+      "grad_norm": 7.971759796142578,
+      "kl": 1.417041015625,
+      "learning_rate": 1.773533563475053e-05,
+      "loss": 0.0567,
+      "reward": 1.32265625,
+      "reward_std": 0.2578751340508461,
+      "rewards/accuracy_reward": 0.39296875,
+      "rewards/format_reward": 0.9296875,
+      "step": 40
+    },
+    {
+      "completion_length": 87.709765625,
+      "epoch": 0.33771106941838647,
+      "grad_norm": 5.786956310272217,
+      "kl": 2.098046875,
+      "learning_rate": 1.6833915640265485e-05,
+      "loss": 0.0839,
+      "reward": 1.275,
+      "reward_std": 0.2750174166634679,
+      "rewards/accuracy_reward": 0.350390625,
+      "rewards/format_reward": 0.924609375,
+      "step": 45
+    },
+    {
+      "completion_length": 55.52890625,
+      "epoch": 0.37523452157598497,
+      "grad_norm": 7.353632926940918,
+      "kl": 2.449560546875,
+      "learning_rate": 1.58135948502146e-05,
+      "loss": 0.098,
+      "reward": 1.349609375,
+      "reward_std": 0.2235970703884959,
+      "rewards/accuracy_reward": 0.39140625,
+      "rewards/format_reward": 0.958203125,
+      "step": 50
+    },
+    {
+      "completion_length": 84.14375,
+      "epoch": 0.41275797373358347,
+      "grad_norm": 5.451722621917725,
+      "kl": 3.009326171875,
+      "learning_rate": 1.4692125452370664e-05,
+      "loss": 0.1204,
+      "reward": 1.3203125,
+      "reward_std": 0.276341262832284,
+      "rewards/accuracy_reward": 0.39296875,
+      "rewards/format_reward": 0.92734375,
+      "step": 55
+    },
+    {
+      "completion_length": 78.859375,
+      "epoch": 0.450281425891182,
+      "grad_norm": 7.11458683013916,
+      "kl": 2.8802734375,
+      "learning_rate": 1.348901948209167e-05,
+      "loss": 0.1152,
+      "reward": 1.30703125,
+      "reward_std": 0.2558631205931306,
+      "rewards/accuracy_reward": 0.38125,
+      "rewards/format_reward": 0.92578125,
+      "step": 60
+    },
+    {
+      "completion_length": 67.624609375,
+      "epoch": 0.4878048780487805,
+      "grad_norm": 1.283034324645996,
+      "kl": 2.932763671875,
+      "learning_rate": 1.2225209339563144e-05,
+      "loss": 0.1172,
+      "reward": 1.30546875,
+      "reward_std": 0.2915836155414581,
+      "rewards/accuracy_reward": 0.371875,
+      "rewards/format_reward": 0.93359375,
+      "step": 65
+    },
+    {
+      "completion_length": 42.2578125,
+      "epoch": 0.525328330206379,
+      "grad_norm": 1.8728909492492676,
+      "kl": 1.6790283203125,
+      "learning_rate": 1.092268359463302e-05,
+      "loss": 0.0672,
+      "reward": 1.354296875,
+      "reward_std": 0.18360752202570438,
+      "rewards/accuracy_reward": 0.38046875,
+      "rewards/format_reward": 0.973828125,
+      "step": 70
+    },
+    {
+      "completion_length": 41.86484375,
+      "epoch": 0.5628517823639775,
+      "grad_norm": 1.411981463432312,
+      "kl": 1.929052734375,
+      "learning_rate": 9.604104415737309e-06,
+      "loss": 0.0771,
+      "reward": 1.363671875,
+      "reward_std": 0.16208343636244535,
+      "rewards/accuracy_reward": 0.388671875,
+      "rewards/format_reward": 0.975,
+      "step": 75
+    },
+    {
+      "completion_length": 42.70625,
+      "epoch": 0.600375234521576,
+      "grad_norm": 116.23101806640625,
+      "kl": 2.564599609375,
+      "learning_rate": 8.292413279130625e-06,
+      "loss": 0.1025,
+      "reward": 1.3796875,
+      "reward_std": 0.14094665497541428,
+      "rewards/accuracy_reward": 0.403515625,
+      "rewards/format_reward": 0.976171875,
+      "step": 80
+    },
+    {
+      "completion_length": 48.035546875,
+      "epoch": 0.6378986866791745,
+      "grad_norm": 3.7647581100463867,
+      "kl": 2.155078125,
+      "learning_rate": 7.010431818542298e-06,
+      "loss": 0.0862,
+      "reward": 1.34609375,
+      "reward_std": 0.19041361436247825,
+      "rewards/accuracy_reward": 0.37890625,
+      "rewards/format_reward": 0.9671875,
+      "step": 85
+    },
+    {
+      "completion_length": 35.946875,
+      "epoch": 0.6754221388367729,
+      "grad_norm": 6.023601055145264,
+      "kl": 1.76474609375,
+      "learning_rate": 5.780464759928623e-06,
+      "loss": 0.0706,
+      "reward": 1.403125,
+      "reward_std": 0.15854822881519795,
+      "rewards/accuracy_reward": 0.423828125,
+      "rewards/format_reward": 0.979296875,
+      "step": 90
+    },
+    {
+      "completion_length": 50.437109375,
+      "epoch": 0.7129455909943715,
+      "grad_norm": 0.8968208432197571,
+      "kl": 2.5580078125,
+      "learning_rate": 4.623911849714226e-06,
+      "loss": 0.1024,
+      "reward": 1.43359375,
+      "reward_std": 0.19160135462880135,
+      "rewards/accuracy_reward": 0.46796875,
+      "rewards/format_reward": 0.965625,
+      "step": 95
+    },
+    {
+      "completion_length": 52.79609375,
+      "epoch": 0.7504690431519699,
+      "grad_norm": 3.7727835178375244,
+      "kl": 2.231298828125,
+      "learning_rate": 3.560895528440844e-06,
+      "loss": 0.0892,
+      "reward": 1.387109375,
+      "reward_std": 0.1720277236774564,
+      "rewards/accuracy_reward": 0.42265625,
+      "rewards/format_reward": 0.964453125,
+      "step": 100
+    },
+    {
+      "epoch": 0.7504690431519699,
+      "eval_completion_length": 79.31458830548927,
+      "eval_kl": 2.734799187052506,
+      "eval_loss": 0.1094130277633667,
+      "eval_reward": 1.933882756563246,
+      "eval_reward_std": 0.12067262680761,
+      "eval_rewards/accuracy_reward": 1.0,
+      "eval_rewards/format_reward": 0.9338827565632458,
+      "eval_runtime": 42224.1278,
+      "eval_samples_per_second": 0.159,
+      "eval_steps_per_second": 0.04,
+      "step": 100
+    },
+    {
+      "completion_length": 58.8546875,
+      "epoch": 0.7879924953095685,
+      "grad_norm": 2.529205322265625,
+      "kl": 2.80966796875,
+      "learning_rate": 2.6099108277934105e-06,
+      "loss": 0.1124,
+      "reward": 1.393359375,
+      "reward_std": 0.2058694703504443,
+      "rewards/accuracy_reward": 0.4375,
+      "rewards/format_reward": 0.955859375,
+      "step": 105
+    },
+    {
+      "completion_length": 41.609765625,
+      "epoch": 0.8255159474671669,
+      "grad_norm": 2.7298059463500977,
+      "kl": 2.009814453125,
+      "learning_rate": 1.7875035823168641e-06,
+      "loss": 0.0804,
+      "reward": 1.428125,
+      "reward_std": 0.1546448241919279,
+      "rewards/accuracy_reward": 0.455078125,
+      "rewards/format_reward": 0.973046875,
+      "step": 110
+    },
+    {
+      "completion_length": 42.476171875,
+      "epoch": 0.8630393996247655,
+      "grad_norm": 2.46567440032959,
+      "kl": 2.199267578125,
+      "learning_rate": 1.1079825545001887e-06,
+      "loss": 0.088,
+      "reward": 1.407421875,
+      "reward_std": 0.163567235134542,
+      "rewards/accuracy_reward": 0.435546875,
+      "rewards/format_reward": 0.971875,
+      "step": 115
+    },
+    {
+      "completion_length": 39.6296875,
+      "epoch": 0.900562851782364,
+      "grad_norm": 0.9802407622337341,
+      "kl": 1.873486328125,
+      "learning_rate": 5.831704818578842e-07,
+      "loss": 0.0749,
+      "reward": 1.403125,
+      "reward_std": 0.14327515792101622,
+      "rewards/accuracy_reward": 0.43125,
+      "rewards/format_reward": 0.971875,
+      "step": 120
+    },
+    {
+      "completion_length": 41.476171875,
+      "epoch": 0.9380863039399625,
+      "grad_norm": 2.266838550567627,
+      "kl": 1.964208984375,
+      "learning_rate": 2.2219837744959284e-07,
+      "loss": 0.0785,
+      "reward": 1.348828125,
+      "reward_std": 0.1762639505788684,
+      "rewards/accuracy_reward": 0.378515625,
+      "rewards/format_reward": 0.9703125,
+      "step": 125
+    },
+    {
+      "completion_length": 42.487890625,
+      "epoch": 0.975609756097561,
+      "grad_norm": 1.1838903427124023,
+      "kl": 2.059765625,
+      "learning_rate": 3.134666272774034e-08,
+      "loss": 0.0824,
+      "reward": 1.376171875,
+      "reward_std": 0.17894832883030176,
+      "rewards/accuracy_reward": 0.4140625,
+      "rewards/format_reward": 0.962109375,
+      "step": 130
+    },
+    {
+      "completion_length": 43.705729166666664,
+      "epoch": 0.99812382739212,
+      "kl": 2.0113932291666665,
+      "reward": 1.3984375,
+      "reward_std": 0.1683385424936811,
+      "rewards/accuracy_reward": 0.4309895833333333,
+      "rewards/format_reward": 0.9674479166666666,
+      "step": 133,
+      "total_flos": 0.0,
+      "train_loss": 0.08425404627709404,
+      "train_runtime": 290681.8756,
+      "train_samples_per_second": 0.059,
+      "train_steps_per_second": 0.0
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 133,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 30,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}