Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,59 @@
+---
+base_model: CohereForAI/aya-expanse-8b
+library_name: peft
+license: other
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+model-index:
+- name: aya-expanse-8b
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# aya-expanse-8b
+
+This model is a fine-tuned version of [CohereForAI/aya-expanse-8b](https://huggingface.co/CohereForAI/aya-expanse-8b) on the combined_train_gemba dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-05
+- train_batch_size: 2
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 4
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 1.0
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.12.0
+- Transformers 4.43.4
+- Pytorch 2.4.0+cu121
+- Datasets 2.20.0
+- Tokenizers 0.19.1

adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "CohereForAI/aya-expanse-8b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c66393ea3ef4f232d80b3017f2271aeb54dbf8bcec5703ac23c301f8e803dd96
+size 109069176

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.999973032011003,
+    "total_flos": 7.616663384064e+17,
+    "train_loss": 0.3924344686242755,
+    "train_runtime": 20071.1732,
+    "train_samples_per_second": 3.695,
+    "train_steps_per_second": 0.924
+}

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<BOS_TOKEN>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|END_OF_TURN_TOKEN|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<PAD>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c69a7ea6c0927dfac8c349186ebcf0466a4723c21cbdb2e850cf559f0bee92b8
+size 12777433

tokenizer_config.json

@@ -0,0 +1,319 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<PAD>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<UNK>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<CLS>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<SEP>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<MASK_TOKEN>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<BOS_TOKEN>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<EOS_TOKEN>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<EOP_TOKEN>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "255000": {
+      "content": "<|START_OF_TURN_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255001": {
+      "content": "<|END_OF_TURN_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "255002": {
+      "content": "<|YES_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255003": {
+      "content": "<|NO_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255004": {
+      "content": "<|GOOD_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255005": {
+      "content": "<|BAD_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255006": {
+      "content": "<|USER_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255007": {
+      "content": "<|CHATBOT_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255008": {
+      "content": "<|SYSTEM_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255009": {
+      "content": "<|USER_0_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255010": {
+      "content": "<|USER_1_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255011": {
+      "content": "<|USER_2_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255012": {
+      "content": "<|USER_3_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255013": {
+      "content": "<|USER_4_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255014": {
+      "content": "<|USER_5_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255015": {
+      "content": "<|USER_6_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255016": {
+      "content": "<|USER_7_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255017": {
+      "content": "<|USER_8_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255018": {
+      "content": "<|USER_9_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255019": {
+      "content": "<|EXTRA_0_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255020": {
+      "content": "<|EXTRA_1_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255021": {
+      "content": "<|EXTRA_2_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255022": {
+      "content": "<|EXTRA_3_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255023": {
+      "content": "<|EXTRA_4_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255024": {
+      "content": "<|EXTRA_5_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255025": {
+      "content": "<|EXTRA_6_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255026": {
+      "content": "<|EXTRA_7_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255027": {
+      "content": "<|EXTRA_8_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "255028": {
+      "content": "<|EXTRA_9_TOKEN|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "bos_token": "<BOS_TOKEN>",
+  "chat_template": "{{ '<BOS_TOKEN>' }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content + '<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}{% elif message['role'] == 'assistant' %}{{ content + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|END_OF_TURN_TOKEN|>",
+  "legacy": true,
+  "merges_file": null,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<PAD>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "CohereTokenizer",
+  "unk_token": null,
+  "use_default_system_prompt": false,
+  "vocab_file": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 0.999973032011003,
+    "total_flos": 7.616663384064e+17,
+    "train_loss": 0.3924344686242755,
+    "train_runtime": 20071.1732,
+    "train_samples_per_second": 3.695,
+    "train_steps_per_second": 0.924
+}

trainer_log.jsonl

@@ -0,0 +1,38 @@
+{"current_steps": 500, "total_steps": 18540, "loss": 0.6808, "learning_rate": 4.9910325182530915e-05, "epoch": 0.026967988997060488, "percentage": 2.7, "elapsed_time": "0:09:28", "remaining_time": "5:42:03", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 1000, "total_steps": 18540, "loss": 0.458, "learning_rate": 4.9641944055954695e-05, "epoch": 0.053935977994120976, "percentage": 5.39, "elapsed_time": "0:09:39", "remaining_time": "2:49:17", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 1500, "total_steps": 18540, "loss": 0.4483, "learning_rate": 4.9196781982554374e-05, "epoch": 0.08090396699118146, "percentage": 8.09, "elapsed_time": "0:19:30", "remaining_time": "3:41:32", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 2000, "total_steps": 18540, "loss": 0.45, "learning_rate": 4.857803254854406e-05, "epoch": 0.10787195598824195, "percentage": 10.79, "elapsed_time": "0:09:27", "remaining_time": "1:18:16", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 2500, "total_steps": 18540, "loss": 0.4444, "learning_rate": 4.7790134653328074e-05, "epoch": 0.13483994498530244, "percentage": 13.48, "elapsed_time": "0:19:13", "remaining_time": "2:03:17", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 3000, "total_steps": 18540, "loss": 0.438, "learning_rate": 4.6838740664901435e-05, "epoch": 0.16180793398236293, "percentage": 16.18, "elapsed_time": "0:29:06", "remaining_time": "2:30:46", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 3500, "total_steps": 18540, "loss": 0.4406, "learning_rate": 4.573067586984441e-05, "epoch": 0.18877592297942342, "percentage": 18.88, "elapsed_time": "0:38:59", "remaining_time": "2:47:33", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 4000, "total_steps": 18540, "loss": 0.4293, "learning_rate": 4.447388950881625e-05, "epoch": 0.2157439119764839, "percentage": 21.57, "elapsed_time": "0:48:45", "remaining_time": "2:57:15", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 4500, "total_steps": 18540, "loss": 0.4347, "learning_rate": 4.307739774881878e-05, "epoch": 0.2427119009735444, "percentage": 24.27, "elapsed_time": "0:58:38", "remaining_time": "3:02:58", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 5000, "total_steps": 18540, "loss": 0.4303, "learning_rate": 4.1551219001346e-05, "epoch": 0.2696798899706049, "percentage": 26.97, "elapsed_time": "1:08:38", "remaining_time": "3:05:54", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 5500, "total_steps": 18540, "loss": 0.4335, "learning_rate": 3.990630205044629e-05, "epoch": 0.29664787896766537, "percentage": 29.67, "elapsed_time": "1:18:27", "remaining_time": "3:06:01", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 6000, "total_steps": 18540, "loss": 0.4307, "learning_rate": 3.815804970461473e-05, "epoch": 0.32361586796472586, "percentage": 32.36, "elapsed_time": "1:28:17", "remaining_time": "3:04:32", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 6500, "total_steps": 18540, "loss": 0.433, "learning_rate": 3.6312001077632294e-05, "epoch": 0.35058385696178634, "percentage": 35.06, "elapsed_time": "1:38:02", "remaining_time": "3:01:36", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 7000, "total_steps": 18540, "loss": 0.4305, "learning_rate": 3.438480032010211e-05, "epoch": 0.37755184595884683, "percentage": 37.76, "elapsed_time": "1:47:59", "remaining_time": "2:58:02", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 7500, "total_steps": 18540, "loss": 0.4292, "learning_rate": 3.2390273142116814e-05, "epoch": 0.4045198349559073, "percentage": 40.45, "elapsed_time": "1:57:58", "remaining_time": "2:53:39", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 8000, "total_steps": 18540, "loss": 0.4271, "learning_rate": 3.034272825252622e-05, "epoch": 0.4314878239529678, "percentage": 43.15, "elapsed_time": "2:07:52", "remaining_time": "2:48:28", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 8500, "total_steps": 18540, "loss": 0.4265, "learning_rate": 2.8256854708469055e-05, "epoch": 0.4584558129500283, "percentage": 45.85, "elapsed_time": "2:17:45", "remaining_time": "2:42:42", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 9000, "total_steps": 18540, "loss": 0.425, "learning_rate": 2.6147616536291464e-05, "epoch": 0.4854238019470888, "percentage": 48.54, "elapsed_time": "2:27:29", "remaining_time": "2:36:20", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 9500, "total_steps": 18540, "loss": 0.4263, "learning_rate": 2.4030145379840563e-05, "epoch": 0.5123917909441493, "percentage": 51.24, "elapsed_time": "2:37:21", "remaining_time": "2:29:44", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 10000, "total_steps": 18540, "loss": 0.4225, "learning_rate": 2.1919631946272402e-05, "epoch": 0.5393597799412098, "percentage": 53.94, "elapsed_time": "2:47:09", "remaining_time": "2:22:45", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 10500, "total_steps": 18540, "loss": 0.4264, "learning_rate": 1.9831217028140688e-05, "epoch": 0.5663277689382703, "percentage": 56.63, "elapsed_time": "2:57:05", "remaining_time": "2:15:35", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 11000, "total_steps": 18540, "loss": 0.4242, "learning_rate": 1.777988288357209e-05, "epoch": 0.5932957579353307, "percentage": 59.33, "elapsed_time": "3:06:56", "remaining_time": "2:08:08", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 11500, "total_steps": 18540, "loss": 0.4223, "learning_rate": 1.578034575376518e-05, "epoch": 0.6202637469323913, "percentage": 62.03, "elapsed_time": "3:16:41", "remaining_time": "2:00:24", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 12000, "total_steps": 18540, "loss": 0.4209, "learning_rate": 1.3850741762328944e-05, "epoch": 0.6472317359294517, "percentage": 64.72, "elapsed_time": "3:26:36", "remaining_time": "1:52:35", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 12500, "total_steps": 18540, "loss": 0.4232, "learning_rate": 1.1997184612520374e-05, "epoch": 0.6741997249265123, "percentage": 67.42, "elapsed_time": "3:36:28", "remaining_time": "1:44:35", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 13000, "total_steps": 18540, "loss": 0.4229, "learning_rate": 1.0236909470428333e-05, "epoch": 0.7011677139235727, "percentage": 70.12, "elapsed_time": "3:46:13", "remaining_time": "1:36:24", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 13500, "total_steps": 18540, "loss": 0.4187, "learning_rate": 8.585739531996178e-06, "epoch": 0.7281357029206332, "percentage": 72.82, "elapsed_time": "3:56:01", "remaining_time": "1:28:07", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 14000, "total_steps": 18540, "loss": 0.4244, "learning_rate": 7.048906317823642e-06, "epoch": 0.7551036919176937, "percentage": 75.51, "elapsed_time": "4:05:52", "remaining_time": "1:19:44", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 14500, "total_steps": 18540, "loss": 0.4178, "learning_rate": 5.640853987596667e-06, "epoch": 0.7820716809147542, "percentage": 78.21, "elapsed_time": "4:15:43", "remaining_time": "1:11:15", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 15000, "total_steps": 18540, "loss": 0.4195, "learning_rate": 4.371683888171277e-06, "epoch": 0.8090396699118146, "percentage": 80.91, "elapsed_time": "4:25:27", "remaining_time": "1:02:38", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 15500, "total_steps": 18540, "loss": 0.4203, "learning_rate": 3.250501027307715e-06, "epoch": 0.8360076589088752, "percentage": 83.6, "elapsed_time": "4:35:07", "remaining_time": "0:53:57", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 16000, "total_steps": 18540, "loss": 0.4187, "learning_rate": 2.287118546736572e-06, "epoch": 0.8629756479059356, "percentage": 86.3, "elapsed_time": "4:44:44", "remaining_time": "0:45:12", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 16500, "total_steps": 18540, "loss": 0.4202, "learning_rate": 1.4845888005343062e-06, "epoch": 0.8899436369029962, "percentage": 89.0, "elapsed_time": "4:54:30", "remaining_time": "0:36:24", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 17000, "total_steps": 18540, "loss": 0.4221, "learning_rate": 8.507582708938533e-07, "epoch": 0.9169116259000566, "percentage": 91.69, "elapsed_time": "5:04:13", "remaining_time": "0:27:33", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 17500, "total_steps": 18540, "loss": 0.4217, "learning_rate": 3.901740487793598e-07, "epoch": 0.9438796148971171, "percentage": 94.39, "elapsed_time": "5:14:03", "remaining_time": "0:18:39", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 18000, "total_steps": 18540, "loss": 0.4209, "learning_rate": 1.0614035867460847e-07, "epoch": 0.9708476038941776, "percentage": 97.09, "elapsed_time": "5:23:49", "remaining_time": "0:09:42", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 18500, "total_steps": 18540, "loss": 0.4205, "learning_rate": 6.94854124816402e-10, "epoch": 0.9978155928912381, "percentage": 99.78, "elapsed_time": "5:33:37", "remaining_time": "0:00:43", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 18540, "total_steps": 18540, "epoch": 0.999973032011003, "percentage": 100.0, "elapsed_time": "5:34:31", "remaining_time": "0:00:00", "throughput": "0.00", "total_tokens": 0}

trainer_state.json

@@ -0,0 +1,301 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.999973032011003,
+  "eval_steps": 500,
+  "global_step": 18540,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.026967988997060488,
+      "grad_norm": 0.8829220533370972,
+      "learning_rate": 4.9910325182530915e-05,
+      "loss": 0.6808,
+      "step": 500
+    },
+    {
+      "epoch": 0.053935977994120976,
+      "grad_norm": 1.0129338502883911,
+      "learning_rate": 4.9641944055954695e-05,
+      "loss": 0.458,
+      "step": 1000
+    },
+    {
+      "epoch": 0.08090396699118146,
+      "grad_norm": 0.8441129326820374,
+      "learning_rate": 4.9196781982554374e-05,
+      "loss": 0.4483,
+      "step": 1500
+    },
+    {
+      "epoch": 0.10787195598824195,
+      "grad_norm": 0.3246585726737976,
+      "learning_rate": 4.857803254854406e-05,
+      "loss": 0.45,
+      "step": 2000
+    },
+    {
+      "epoch": 0.13483994498530244,
+      "grad_norm": 0.44776633381843567,
+      "learning_rate": 4.7790134653328074e-05,
+      "loss": 0.4444,
+      "step": 2500
+    },
+    {
+      "epoch": 0.16180793398236293,
+      "grad_norm": 0.49329063296318054,
+      "learning_rate": 4.6838740664901435e-05,
+      "loss": 0.438,
+      "step": 3000
+    },
+    {
+      "epoch": 0.18877592297942342,
+      "grad_norm": 0.47724494338035583,
+      "learning_rate": 4.573067586984441e-05,
+      "loss": 0.4406,
+      "step": 3500
+    },
+    {
+      "epoch": 0.2157439119764839,
+      "grad_norm": 0.5984110236167908,
+      "learning_rate": 4.447388950881625e-05,
+      "loss": 0.4293,
+      "step": 4000
+    },
+    {
+      "epoch": 0.2427119009735444,
+      "grad_norm": 0.29806196689605713,
+      "learning_rate": 4.307739774881878e-05,
+      "loss": 0.4347,
+      "step": 4500
+    },
+    {
+      "epoch": 0.2696798899706049,
+      "grad_norm": 0.5836197137832642,
+      "learning_rate": 4.1551219001346e-05,
+      "loss": 0.4303,
+      "step": 5000
+    },
+    {
+      "epoch": 0.29664787896766537,
+      "grad_norm": 0.35174188017845154,
+      "learning_rate": 3.990630205044629e-05,
+      "loss": 0.4335,
+      "step": 5500
+    },
+    {
+      "epoch": 0.32361586796472586,
+      "grad_norm": 0.529833197593689,
+      "learning_rate": 3.815804970461473e-05,
+      "loss": 0.4307,
+      "step": 6000
+    },
+    {
+      "epoch": 0.35058385696178634,
+      "grad_norm": 0.2829456329345703,
+      "learning_rate": 3.6312001077632294e-05,
+      "loss": 0.433,
+      "step": 6500
+    },
+    {
+      "epoch": 0.37755184595884683,
+      "grad_norm": 0.3882625102996826,
+      "learning_rate": 3.438480032010211e-05,
+      "loss": 0.4305,
+      "step": 7000
+    },
+    {
+      "epoch": 0.4045198349559073,
+      "grad_norm": 0.4895012378692627,
+      "learning_rate": 3.2390273142116814e-05,
+      "loss": 0.4292,
+      "step": 7500
+    },
+    {
+      "epoch": 0.4314878239529678,
+      "grad_norm": 0.4984913468360901,
+      "learning_rate": 3.034272825252622e-05,
+      "loss": 0.4271,
+      "step": 8000
+    },
+    {
+      "epoch": 0.4584558129500283,
+      "grad_norm": 0.4299156367778778,
+      "learning_rate": 2.8256854708469055e-05,
+      "loss": 0.4265,
+      "step": 8500
+    },
+    {
+      "epoch": 0.4854238019470888,
+      "grad_norm": 0.44763749837875366,
+      "learning_rate": 2.6147616536291464e-05,
+      "loss": 0.425,
+      "step": 9000
+    },
+    {
+      "epoch": 0.5123917909441493,
+      "grad_norm": 0.6412510275840759,
+      "learning_rate": 2.4030145379840563e-05,
+      "loss": 0.4263,
+      "step": 9500
+    },
+    {
+      "epoch": 0.5393597799412098,
+      "grad_norm": 0.884003758430481,
+      "learning_rate": 2.1919631946272402e-05,
+      "loss": 0.4225,
+      "step": 10000
+    },
+    {
+      "epoch": 0.5663277689382703,
+      "grad_norm": 0.4416724443435669,
+      "learning_rate": 1.9831217028140688e-05,
+      "loss": 0.4264,
+      "step": 10500
+    },
+    {
+      "epoch": 0.5932957579353307,
+      "grad_norm": 0.5343862175941467,
+      "learning_rate": 1.777988288357209e-05,
+      "loss": 0.4242,
+      "step": 11000
+    },
+    {
+      "epoch": 0.6202637469323913,
+      "grad_norm": 0.5542997717857361,
+      "learning_rate": 1.578034575376518e-05,
+      "loss": 0.4223,
+      "step": 11500
+    },
+    {
+      "epoch": 0.6472317359294517,
+      "grad_norm": 0.8904162049293518,
+      "learning_rate": 1.3850741762328944e-05,
+      "loss": 0.4209,
+      "step": 12000
+    },
+    {
+      "epoch": 0.6741997249265123,
+      "grad_norm": 0.4040282368659973,
+      "learning_rate": 1.1997184612520374e-05,
+      "loss": 0.4232,
+      "step": 12500
+    },
+    {
+      "epoch": 0.7011677139235727,
+      "grad_norm": 0.5565162897109985,
+      "learning_rate": 1.0236909470428333e-05,
+      "loss": 0.4229,
+      "step": 13000
+    },
+    {
+      "epoch": 0.7281357029206332,
+      "grad_norm": 0.6723429560661316,
+      "learning_rate": 8.585739531996178e-06,
+      "loss": 0.4187,
+      "step": 13500
+    },
+    {
+      "epoch": 0.7551036919176937,
+      "grad_norm": 0.35649460554122925,
+      "learning_rate": 7.048906317823642e-06,
+      "loss": 0.4244,
+      "step": 14000
+    },
+    {
+      "epoch": 0.7820716809147542,
+      "grad_norm": 0.3733614683151245,
+      "learning_rate": 5.640853987596667e-06,
+      "loss": 0.4178,
+      "step": 14500
+    },
+    {
+      "epoch": 0.8090396699118146,
+      "grad_norm": 0.889401912689209,
+      "learning_rate": 4.371683888171277e-06,
+      "loss": 0.4195,
+      "step": 15000
+    },
+    {
+      "epoch": 0.8360076589088752,
+      "grad_norm": 0.4235946536064148,
+      "learning_rate": 3.250501027307715e-06,
+      "loss": 0.4203,
+      "step": 15500
+    },
+    {
+      "epoch": 0.8629756479059356,
+      "grad_norm": 0.5250910520553589,
+      "learning_rate": 2.287118546736572e-06,
+      "loss": 0.4187,
+      "step": 16000
+    },
+    {
+      "epoch": 0.8899436369029962,
+      "grad_norm": 0.5213350057601929,
+      "learning_rate": 1.4845888005343062e-06,
+      "loss": 0.4202,
+      "step": 16500
+    },
+    {
+      "epoch": 0.9169116259000566,
+      "grad_norm": 0.4989272356033325,
+      "learning_rate": 8.507582708938533e-07,
+      "loss": 0.4221,
+      "step": 17000
+    },
+    {
+      "epoch": 0.9438796148971171,
+      "grad_norm": 0.4094185531139374,
+      "learning_rate": 3.901740487793598e-07,
+      "loss": 0.4217,
+      "step": 17500
+    },
+    {
+      "epoch": 0.9708476038941776,
+      "grad_norm": 0.5710214972496033,
+      "learning_rate": 1.0614035867460847e-07,
+      "loss": 0.4209,
+      "step": 18000
+    },
+    {
+      "epoch": 0.9978155928912381,
+      "grad_norm": 0.5618287920951843,
+      "learning_rate": 6.94854124816402e-10,
+      "loss": 0.4205,
+      "step": 18500
+    },
+    {
+      "epoch": 0.999973032011003,
+      "step": 18540,
+      "total_flos": 7.616663384064e+17,
+      "train_loss": 0.3924344686242755,
+      "train_runtime": 20071.1732,
+      "train_samples_per_second": 3.695,
+      "train_steps_per_second": 0.924
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 18540,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.616663384064e+17,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f439b6f67a24a2bdf25a68b75e5fe1e8c471ef66c53f9032914a71e66a1621c
+size 5560