Upload 13 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+bge-en-icl-paddle/lora_model_state.pdparams filter=lfs diff=lfs merge=lfs -text

bge-en-icl-paddle/added_tokens.json

@@ -0,0 +1 @@
+{"<unk>": 0, "<s>": 1, "</s>": 2, "<instruct>": 32000, "<query>": 32001, "<response>": 32002}

bge-en-icl-paddle/all_results.json

@@ -0,0 +1,7 @@
+{
+    "progress_or_epoch": 0.064,
+    "train_loss": 0.20930041134357452,
+    "train_runtime": 34189.7588,
+    "train_samples_per_second": 0.3744,
+    "train_steps_per_second": 0.0029
+}

bge-en-icl-paddle/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "hf_models/bge-en-icl",
+  "architectures": [
+    "MistralModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "paddlenlp_version": "3.0.0b4.post20250825",
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "vocab_size": 32003
+}

bge-en-icl-paddle/lora_config.json

@@ -0,0 +1,28 @@
+{
+  "base_model_name_or_path": null,
+  "do_qat": false,
+  "dtype": "float16",
+  "enable_lora_list": null,
+  "head_dim": null,
+  "lora_alpha": 64,
+  "lora_dropout": 0.1,
+  "lora_plus_scale": 1.0,
+  "lora_use_mixer": false,
+  "loraga": false,
+  "lorapro": false,
+  "merge_weights": false,
+  "pissa": false,
+  "r": 32,
+  "rslora": false,
+  "scaling": 2.0,
+  "target_modules": [
+    ".*q_proj$",
+    ".*k_proj$",
+    ".*v_proj$"
+  ],
+  "tensor_parallel_degree": 1,
+  "trainable_bias": null,
+  "trainable_modules": null,
+  "use_mora": false,
+  "use_quick_lora": false
+}

bge-en-icl-paddle/lora_model_state.pdparams

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e86bd35dc92792eacfacec7f91ab64407694fc1a9e3b56f74b9581c460bda96
+size 113314622

bge-en-icl-paddle/sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

bge-en-icl-paddle/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "pad_token": "<unk>", "additional_special_tokens": [{"content": "<instruct>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"content": "<query>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"content": "<response>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}]}

bge-en-icl-paddle/tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_bos_token": true, "add_eos_token": true, "sp_model_kwargs": {}, "decode_with_prefix_space": false, "add_prefix_space": true, "added_tokens_decoder": {"0": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "1": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "2": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32000": {"content": "<instruct>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32001": {"content": "<query>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32002": {"content": "<response>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}}, "additional_special_tokens": ["<instruct>", "<query>", "<response>"], "clean_up_tokenization_spaces": false, "legacy": true, "model_max_length": 1000000000000000019884624838656, "pad_token": "<unk>", "spaces_between_special_tokens": false, "use_default_system_prompt": false, "name_or_path": "/mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle", "from_slow": true, "_from_auto": true, "special_tokens_map_file": "/mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle/special_tokens_map.json", "tokenizer_class": "LlamaTokenizer"}

bge-en-icl-paddle/train_bge-en-icl-paddle_mmarco.jsonl.log

@@ -0,0 +1,745 @@
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md
+  warnings.warn(warning_message)
+LAUNCH INFO 2025-11-02 02:43:59,501 -----------  Configuration  ----------------------
+LAUNCH INFO 2025-11-02 02:43:59,501 auto_cluster_config: 0
+LAUNCH INFO 2025-11-02 02:43:59,501 auto_parallel_config: None
+LAUNCH INFO 2025-11-02 02:43:59,501 auto_tuner_json: None
+LAUNCH INFO 2025-11-02 02:43:59,501 devices: 0,1,2,3
+LAUNCH INFO 2025-11-02 02:43:59,501 elastic_level: -1
+LAUNCH INFO 2025-11-02 02:43:59,501 elastic_timeout: 30
+LAUNCH INFO 2025-11-02 02:43:59,501 enable_gpu_log: True
+LAUNCH INFO 2025-11-02 02:43:59,501 gloo_port: 6767
+LAUNCH INFO 2025-11-02 02:43:59,501 host: None
+LAUNCH INFO 2025-11-02 02:43:59,501 ips: None
+LAUNCH INFO 2025-11-02 02:43:59,501 job_id: default
+LAUNCH INFO 2025-11-02 02:43:59,501 legacy: False
+LAUNCH INFO 2025-11-02 02:43:59,515 log_dir: log
+LAUNCH INFO 2025-11-02 02:43:59,515 log_level: INFO
+LAUNCH INFO 2025-11-02 02:43:59,515 log_overwrite: False
+LAUNCH INFO 2025-11-02 02:43:59,515 master: None
+LAUNCH INFO 2025-11-02 02:43:59,515 max_restart: 3
+LAUNCH INFO 2025-11-02 02:43:59,515 nnodes: 1
+LAUNCH INFO 2025-11-02 02:43:59,515 nproc_per_node: None
+LAUNCH INFO 2025-11-02 02:43:59,515 rank: -1
+LAUNCH INFO 2025-11-02 02:43:59,515 run_mode: collective
+LAUNCH INFO 2025-11-02 02:43:59,515 server_num: None
+LAUNCH INFO 2025-11-02 02:43:59,515 servers: 
+LAUNCH INFO 2025-11-02 02:43:59,515 sort_ip: False
+LAUNCH INFO 2025-11-02 02:43:59,515 start_port: 6070
+LAUNCH INFO 2025-11-02 02:43:59,542 trainer_num: None
+LAUNCH INFO 2025-11-02 02:43:59,542 trainers: 
+LAUNCH INFO 2025-11-02 02:43:59,542 training_script: train.py
+LAUNCH INFO 2025-11-02 02:43:59,542 training_script_args: ['--do_train', '--query_instruction_for_retrieval', 'query: ', '--passage_instruction_for_retrieval', '', '--model_name_or_path', '/mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle', '--output_dir', 'tmp_train_mmarco.jsonl/bge-en-icl-paddle', '--save_steps', '999999999', '--train_data', './data/mmarco.jsonl', '--fp16_opt_level', 'O2', '--fp16', '--per_device_train_batch_size', '1', '--gradient_accumulation_steps', '32', '--recompute', '--train_group_size', '4', '--learning_rate', '1e-4', '--query_max_len', '128', '--passage_max_len', '4096', '--num_train_epochs', '1', '--logging_steps', '1', '--overwrite_output_dir', '--negatives_cross_device', '--warmup_steps', '10', '--max_steps', '100', '--do_train', '--fine_tune_type', 'lora', '--sentence_pooling_method', 'last', '--sharding', 'stage3 offload', '--use_flash_attention', '--temperature', '0.01']
+LAUNCH INFO 2025-11-02 02:43:59,542 with_gloo: 1
+LAUNCH INFO 2025-11-02 02:43:59,542 --------------------------------------------------
+LAUNCH INFO 2025-11-02 02:43:59,569 Job: default, mode collective, replicas 1[1:1], elastic False
+LAUNCH INFO 2025-11-02 02:43:59,648 Run Pod: cbsudf, replicas 4, status ready
+LAUNCH INFO 2025-11-02 02:44:00,083 Watching Pod: cbsudf, replicas 4, status running
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md
+  warnings.warn(warning_message)
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/_distutils_hack/__init__.py:30: UserWarning: Setuptools is replacing distutils. Support for replacing an already imported distutils is deprecated. In the future, this condition will fail. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddlenlp/trainer/training_args.py:1205: UserWarning: `offload` is not supported NOW!
+  warnings.warn("`offload` is not supported NOW!")
+[2025-11-02 02:44:04,759] [    INFO] distributed_strategy.py:335 - distributed strategy initialized
+======================= Modified FLAGS detected =======================
+FLAGS(name='FLAGS_cusolver_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cusolver/lib', default_value='')
+FLAGS(name='FLAGS_selected_gpus', current_value='0', default_value='')
+FLAGS(name='FLAGS_cusparse_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cusparse/lib', default_value='')
+FLAGS(name='FLAGS_cupti_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cuda_cupti/lib', default_value='')
+FLAGS(name='FLAGS_curand_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/curand/lib', default_value='')
+FLAGS(name='FLAGS_enable_pir_in_executor', current_value=True, default_value=False)
+FLAGS(name='FLAGS_nccl_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/nccl/lib', default_value='')
+FLAGS(name='FLAGS_cudnn_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cudnn/lib', default_value='')
+FLAGS(name='FLAGS_cublas_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cublas/lib', default_value='')
+FLAGS(name='FLAGS_nvidia_package_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia', default_value='')
+FLAGS(name='FLAGS_cuda_cccl_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cuda_cccl/include/', default_value='')
+=======================================================================
+I1102 02:44:04.761015 76025 tcp_store.cc:336] create libuv server at port: 44351
+I1102 02:44:04.761771 76025 tcp_utils.cc:132] Successfully connected to 172.17.0.5:44351
+I1102 02:44:05.356057 76025 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1102 02:44:05.356122 76025 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-02 02:44:05,356] [    INFO] topology.py:526 - Total 4 pipe comm group(s) create successfully!
+W1102 02:44:05.686445 76025 gpu_resources.cc:114] Please NOTE: device: 0, GPU Compute Capability: 8.0, Driver API Version: 12.8, Runtime API Version: 12.8
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/distributed/communication/group.py:145: UserWarning: Current global rank 0 is not in group _default_pg10
+  warnings.warn(
+[2025-11-02 02:44:05,691] [    INFO] topology.py:526 - Total 4 data comm group(s) create successfully!
+[2025-11-02 02:44:05,691] [    INFO] topology.py:526 - Total 4 model comm group(s) create successfully!
+I1102 02:44:05.692106 76025 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1102 02:44:05.692123 76025 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-02 02:44:05,692] [    INFO] topology.py:526 - Total 1 sharding comm group(s) create successfully!
+I1102 02:44:05.692240 76025 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1102 02:44:05.692250 76025 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-02 02:44:05,692] [    INFO] topology.py:440 - HybridParallelInfo: rank_id: 0, mp_degree: 1, sharding_degree: 4, pp_degree: 1, dp_degree: 1, sep_degree: 1, mp_group: [0],  sharding_group: [0, 1, 2, 3], pp_group: [0], dp_group: [0], sep:group: None, check/clip group: [0, 1, 2, 3]
+[2025-11-02 02:44:05,692] [    INFO] -     +==============================================================================+
+    |                                                                              |
+    |                         DistributedStrategy Overview                         |
+    |                                                                              |
+    +==============================================================================+
+    |                        a_sync=True <-> a_sync_configs                        |
+    +------------------------------------------------------------------------------+
+    |                               k_steps                    -1                  |
+    |                     max_merge_var_num                    1                   |
+    |                       send_queue_size                    16                  |
+    |               independent_recv_thread                  False                 |
+    |         min_send_grad_num_before_recv                    1                   |
+    |                      thread_pool_size                    1                   |
+    |                       send_wait_times                    1                   |
+    |               runtime_split_send_recv                  False                 |
+    |                        launch_barrier                   True                 |
+    |             heter_worker_device_guard                   cpu                  |
+    |                        lr_decay_steps                    10                  |
+    |                            use_ps_gpu                    0                   |
+    |                         use_gpu_graph                    0                   |
+    +==============================================================================+
+    |                    Environment Flags, Communication Flags                    |
+    +------------------------------------------------------------------------------+
+    |                                  mode                    1                   |
+    |                               elastic                  False                 |
+    |                                  auto                  False                 |
+    |                   sync_nccl_allreduce                   True                 |
+    |                         nccl_comm_num                    1                   |
+    |            use_hierarchical_allreduce                  False                 |
+    |   hierarchical_allreduce_inter_nranks                    1                   |
+    |                       sync_batch_norm                  False                 |
+    |                   fuse_all_reduce_ops                   True                 |
+    |                  fuse_grad_size_in_MB                    32                  |
+    |              fuse_grad_size_in_TFLOPS                   50.0                 |
+    |               cudnn_exhaustive_search                  False                 |
+    |             conv_workspace_size_limit                   512                  |
+    |    cudnn_batchnorm_spatial_persistent                  False                 |
+    |                        fp16_allreduce                  False                 |
+    |               last_comm_group_size_MB                   1.0                  |
+    |                find_unused_parameters                  False                 |
+    |            without_graph_optimization                   True                 |
+    |                 fuse_grad_size_in_num                    8                   |
+    |                 calc_comm_same_stream                  False                 |
+    |                                   asp                  False                 |
+    |                       fuse_grad_merge                  False                 |
+    |                             semi_auto                  False                 |
+    |                            adam_d2sum                  False                 |
+    |                           auto_search                  False                 |
+    |                        heter_ccl_mode                  False                 |
+    |                         is_fl_ps_mode                  False                 |
+    |                      with_coordinator                  False                 |
+    |                            split_data                   True                 |
+    |                  downpour_table_param                    []                  |
+    |                       fs_client_param                                        |
+    +==============================================================================+
+    |                                Build Strategy                                |
+    +------------------------------------------------------------------------------+
+    |              fuse_elewise_add_act_ops                  False                 |
+    |                       fuse_bn_act_ops                  False                 |
+    |              fuse_relu_depthwise_conv                  False                 |
+    |                    fuse_broadcast_ops                  False                 |
+    |                fuse_all_optimizer_ops                  False                 |
+    |                        enable_inplace                  False                 |
+    |     enable_backward_optimizer_op_deps                   True                 |
+    |                 cache_runtime_context                  False                 |
+    |                   fuse_bn_add_act_ops                   True                 |
+    |                    enable_auto_fusion                  False                 |
+    |                          enable_addto                  False                 |
+    |              allow_cuda_graph_capture                  False                 |
+    |                       reduce_strategy                    0                   |
+    |                    fuse_gemm_epilogue                  False                 |
+    |                   debug_graphviz_path                                        |
+    |                       fused_attention                  False                 |
+    |                     fused_feedforward                  False                 |
+    |            fuse_dot_product_attention                  False                 |
+    |                          fuse_resunit                  False                 |
+    +==============================================================================+
+
+[2025-11-02 02:44:05,693] [    INFO] - The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).
+[2025-11-02 02:44:05,693] [ WARNING] - Process rank: 0, device: gpu, distributed training: True, 16-bits training: True
+[2025-11-02 02:44:05,694] [    INFO] - Training/evaluation parameters RetrieverTrainingArguments(
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+amp_custom_black_list=None,
+amp_custom_white_list=None,
+amp_master_grad=False,
+auto_parallel_resume_form_hybrid_parallel=False,
+bf16=False,
+bf16_full_eval=False,
+ckpt_quant_stage=O0,
+context_parallel_degree=1,
+count_trained_tokens=False,
+data_parallel_config=,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_shuffle=True,
+ddp_find_unused_parameters=None,
+device=gpu,
+disable_tqdm=False,
+distributed_dataloader=False,
+do_eval=False,
+do_export=False,
+do_predict=False,
+do_train=True,
+enable_auto_parallel=False,
+enable_zero_cost_checkpoint=False,
+eval_accumulation_steps=None,
+eval_steps=None,
+evaluation_strategy=IntervalStrategy.NO,
+expert_max_capacity=4294967296,
+expert_min_capacity=1,
+expert_parallel_degree=1,
+expert_tensor_parallel_degree=1,
+fine_tune_type=lora,
+fix_position_embedding=False,
+flash_device_save_steps=0,
+flatten_param_grads=False,
+force_reshard_pp=False,
+fp16=True,
+fp16_full_eval=False,
+fp16_opt_level=O2,
+fuse_sequence_parallel_allreduce=False,
+gradient_accumulation_steps=32,
+greater_is_better=None,
+hybrid_parallel_topo_order=pp_first,
+ignore_data_skip=False,
+ignore_load_lr_and_optim=False,
+ignore_save_lr_and_optim=False,
+label_names=None,
+lazy_data_processing=True,
+learning_rate=0.0001,
+load_best_model_at_end=False,
+load_sharded_model=False,
+local_rank=0,
+log_on_each_node=True,
+logging_dir=tmp_train_mmarco.jsonl/bge-en-icl-paddle/runs/Nov02_02-44-04_b63e119a1648,
+logging_first_step=False,
+logging_steps=1,
+logging_strategy=IntervalStrategy.STEPS,
+lr_end=1e-07,
+lr_scheduler_type=SchedulerType.LINEAR,
+margin=0.2,
+matryoshka_dims=[64, 128, 256, 512, 768],
+matryoshka_loss_weights=[1, 1, 1, 1, 1],
+max_evaluate_steps=-1,
+max_grad_norm=1.0,
+max_steps=100,
+metric_for_best_model=None,
+metrics_output_path=None,
+min_lr=0.0,
+minimum_eval_times=None,
+nccl_comm_group_config=None,
+negatives_cross_device=True,
+no_cuda=False,
+num_cycles=0.5,
+num_train_epochs=1.0,
+offload_optim=False,
+optim=OptimizerNames.ADAMW,
+ordered_save_group_size=0,
+output_dir=tmp_train_mmarco.jsonl/bge-en-icl-paddle,
+output_signal_dir=tmp_train_mmarco.jsonl/bge-en-icl-paddle,
+overwrite_output_dir=True,
+pad_token_id=0,
+past_index=-1,
+pdc_download_ckpt=False,
+pdc_download_timeout=300,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+pipeline_parallel_config=,
+pipeline_parallel_degree=1,
+power=1.0,
+prediction_loss_only=False,
+recompute=True,
+refined_recompute={},
+release_grads=False,
+remove_unused_columns=True,
+report_to=['visualdl'],
+resume_from_checkpoint=None,
+run_name=tmp_train_mmarco.jsonl/bge-en-icl-paddle,
+save_on_each_node=False,
+save_rng_states=True,
+save_sharded_model=False,
+save_sharding_stage1_model_include_freeze_params=False,
+save_steps=999999999,
+save_strategy=IntervalStrategy.STEPS,
+save_tokenizer=True,
+save_total_limit=None,
+scale_loss=32768,
+seed=42,
+sentence_pooling_method=last,
+sep_parallel_degree=1,
+sequence_parallel=False,
+sequence_parallel_config=,
+sharding=[<ShardingOption.FULL_SHARD: 'stage3'>, <ShardingOption.OFFLOAD: 'offload'>],
+sharding_comm_buffer_size_MB=-1,
+sharding_degree=-1,
+sharding_parallel_config=,
+sharding_parallel_degree=4,
+sharding_parallel_mesh_dimension=dp,
+skip_data_intervals=None,
+skip_memory_metrics=True,
+skip_profile_timer=True,
+split_inputs_sequence_dim=True,
+split_norm_comm=False,
+temperature=0.01,
+tensor_parallel_config=,
+tensor_parallel_degree=1,
+tensorwise_offload_optimizer=False,
+to_static=False,
+unified_checkpoint=False,
+unified_checkpoint_config=,
+use_async_save=False,
+use_expert_parallel=False,
+use_inbatch_neg=False,
+use_lowprecision_moment=False,
+use_matryoshka=False,
+wandb_api_key=None,
+wandb_http_proxy=None,
+warmup_ratio=0.0,
+warmup_steps=10,
+weight_decay=0.0,
+zcc_ema_interval=1,
+zcc_pipeline_hooks_capacity_usage=0.6,
+zcc_save_ema_coef=None,
+zcc_workers_num=3,
+)
+[2025-11-02 02:44:05,694] [    INFO] - Model parameters ModelArguments(model_name_or_path='/mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle', tokenizer_name=None, normalized=True, use_flash_attention=True)
+[2025-11-02 02:44:05,695] [    INFO] - Data parameters DataArguments(train_data='./data/mmarco.jsonl', train_group_size=4, query_max_len=128, passage_max_len=4096, max_example_num_per_dataset=100000000, query_instruction_for_retrieval='query: ', passage_instruction_for_retrieval='')
+[2025-11-02 02:44:05,697] [    INFO] - The global seed is set to 42, local seed is set to 46 and random seed is set to 42.
+[2025-11-02 02:44:05,716] [    INFO] - We are using <class 'paddlenlp.transformers.mistral.modeling.MistralModel'> to load '/mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle'.
+[2025-11-02 02:44:05,717] [    INFO] - Loading configuration file /mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle/config.json
+[2025-11-02 02:44:05,721] [    INFO] - Loading weights file /mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle/model_state.pdparams.index.json
+file: /mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle/model_state.pdparams.index.json is paddle weight.
+
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:11<01:17, 11.13s/it]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:22<01:06, 11.11s/it]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:33<00:55, 11.14s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:44<00:43, 10.98s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:55<00:33, 11.03s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [01:05<00:21, 10.89s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [01:16<00:10, 10.93s/it]
+Loading checkpoint shards: 100%|██████████| 8/8 [01:20<00:00,  8.48s/it]
+Loading checkpoint shards: 100%|██████████| 8/8 [01:20<00:00, 10.01s/it]
+[2025-11-02 02:45:26,258] [    INFO] - All model checkpoint weights were used when initializing MistralModel.
+
+[2025-11-02 02:45:26,260] [    INFO] - All the weights of MistralModel were initialized from the model checkpoint at /mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use MistralModel for predictions without further training.
+[2025-11-02 02:45:26,263] [    INFO] - Loading configuration file /mnt/141nfs/lizhuoqun/hf_models/bge-en-icl-paddle/config.json
+[2025-11-02 02:45:26,438] [ WARNING] - Reset tensor_parallel_degree of lora_config to 1.
+[2025-11-02 02:45:26,438] [    INFO] - Mark only lora and trainable_module as trainable.
+[2025-11-02 02:45:26,459] [   DEBUG] - Frozen parameters: 2.13e+10 || Trainable parameters:5.66e+07 || Total parameters:2.14e+10|| Trainable:0.26%
+[2025-11-02 02:45:26,497] [    INFO] - The global seed is set to 42, local seed is set to 46 and random seed is set to 42.
+[2025-11-02 02:45:26,553] [    INFO] - max_steps is given, it will override any value given in num_train_epochs
+[2025-11-02 02:45:26,553] [    INFO] - Using half precision
+[2025-11-02 02:45:26,573] [   DEBUG] - ============================================================
+[2025-11-02 02:45:26,573] [   DEBUG] -     Training Configuration Arguments    
+[2025-11-02 02:45:26,574] [   DEBUG] - paddle commit id              : dc312219f1498234f178a3b6a93e9f1f4ddebcb8
+[2025-11-02 02:45:26,574] [   DEBUG] - paddlenlp commit id           : 0bb61071d051adf6207199d6493b08079d58f8b2.dirty
+[2025-11-02 02:45:26,574] [   DEBUG] - _no_sync_in_gradient_accumulation: True
+[2025-11-02 02:45:26,574] [   DEBUG] - adam_beta1                    : 0.9
+[2025-11-02 02:45:26,574] [   DEBUG] - adam_beta2                    : 0.999
+[2025-11-02 02:45:26,574] [   DEBUG] - adam_epsilon                  : 1e-08
+[2025-11-02 02:45:26,574] [   DEBUG] - amp_custom_black_list         : None
+[2025-11-02 02:45:26,574] [   DEBUG] - amp_custom_white_list         : None
+[2025-11-02 02:45:26,574] [   DEBUG] - amp_master_grad               : False
+[2025-11-02 02:45:26,574] [   DEBUG] - auto_parallel_resume_form_hybrid_parallel: False
+[2025-11-02 02:45:26,574] [   DEBUG] - bf16                          : False
+[2025-11-02 02:45:26,574] [   DEBUG] - bf16_full_eval                : False
+[2025-11-02 02:45:26,574] [   DEBUG] - ckpt_quant_stage              : O0
+[2025-11-02 02:45:26,574] [   DEBUG] - context_parallel_degree       : 1
+[2025-11-02 02:45:26,574] [   DEBUG] - count_trained_tokens          : False
+[2025-11-02 02:45:26,574] [   DEBUG] - current_device                : gpu:0
+[2025-11-02 02:45:26,574] [   DEBUG] - data_parallel_config          : 
+[2025-11-02 02:45:26,575] [   DEBUG] - data_parallel_degree          : 1
+[2025-11-02 02:45:26,575] [   DEBUG] - data_parallel_rank            : 0
+[2025-11-02 02:45:26,575] [   DEBUG] - dataloader_drop_last          : False
+[2025-11-02 02:45:26,575] [   DEBUG] - dataloader_num_workers        : 0
+[2025-11-02 02:45:26,575] [   DEBUG] - dataloader_shuffle            : True
+[2025-11-02 02:45:26,575] [   DEBUG] - dataset_rank                  : 0
+[2025-11-02 02:45:26,575] [   DEBUG] - dataset_world_size            : 4
+[2025-11-02 02:45:26,575] [   DEBUG] - ddp_find_unused_parameters    : None
+[2025-11-02 02:45:26,575] [   DEBUG] - device                        : gpu
+[2025-11-02 02:45:26,575] [   DEBUG] - disable_tqdm                  : False
+[2025-11-02 02:45:26,575] [   DEBUG] - distributed_dataloader        : False
+[2025-11-02 02:45:26,575] [   DEBUG] - do_eval                       : False
+[2025-11-02 02:45:26,575] [   DEBUG] - do_export                     : False
+[2025-11-02 02:45:26,575] [   DEBUG] - do_predict                    : False
+[2025-11-02 02:45:26,575] [   DEBUG] - do_train                      : True
+[2025-11-02 02:45:26,575] [   DEBUG] - enable_auto_parallel          : False
+[2025-11-02 02:45:26,575] [   DEBUG] - enable_zero_cost_checkpoint   : False
+[2025-11-02 02:45:26,575] [   DEBUG] - eval_accumulation_steps       : None
+[2025-11-02 02:45:26,575] [   DEBUG] - eval_batch_size               : 8
+[2025-11-02 02:45:26,576] [   DEBUG] - eval_steps                    : None
+[2025-11-02 02:45:26,576] [   DEBUG] - evaluation_strategy           : IntervalStrategy.NO
+[2025-11-02 02:45:26,576] [   DEBUG] - expert_max_capacity           : 4294967296
+[2025-11-02 02:45:26,576] [   DEBUG] - expert_min_capacity           : 1
+[2025-11-02 02:45:26,576] [   DEBUG] - expert_parallel_degree        : 1
+[2025-11-02 02:45:26,576] [   DEBUG] - expert_tensor_parallel_degree : 1
+[2025-11-02 02:45:26,576] [   DEBUG] - fine_tune_type                : lora
+[2025-11-02 02:45:26,576] [   DEBUG] - fix_position_embedding        : False
+[2025-11-02 02:45:26,576] [   DEBUG] - flash_device_save_steps       : 0
+[2025-11-02 02:45:26,576] [   DEBUG] - flatten_param_grads           : False
+[2025-11-02 02:45:26,576] [   DEBUG] - force_reshard_pp              : False
+[2025-11-02 02:45:26,576] [   DEBUG] - fp16                          : True
+[2025-11-02 02:45:26,576] [   DEBUG] - fp16_full_eval                : False
+[2025-11-02 02:45:26,576] [   DEBUG] - fp16_opt_level                : O2
+[2025-11-02 02:45:26,576] [   DEBUG] - fuse_sequence_parallel_allreduce: False
+[2025-11-02 02:45:26,576] [   DEBUG] - gradient_accumulation_steps   : 32
+[2025-11-02 02:45:26,576] [   DEBUG] - greater_is_better             : None
+[2025-11-02 02:45:26,576] [   DEBUG] - hybrid_parallel_topo_order    : pp_first
+[2025-11-02 02:45:26,576] [   DEBUG] - ignore_data_skip              : False
+[2025-11-02 02:45:26,576] [   DEBUG] - ignore_load_lr_and_optim      : False
+[2025-11-02 02:45:26,577] [   DEBUG] - ignore_save_lr_and_optim      : False
+[2025-11-02 02:45:26,577] [   DEBUG] - label_names                   : None
+[2025-11-02 02:45:26,577] [   DEBUG] - lazy_data_processing          : True
+[2025-11-02 02:45:26,577] [   DEBUG] - learning_rate                 : 0.0001
+[2025-11-02 02:45:26,577] [   DEBUG] - load_best_model_at_end        : False
+[2025-11-02 02:45:26,577] [   DEBUG] - load_sharded_model            : False
+[2025-11-02 02:45:26,577] [   DEBUG] - local_process_index           : 0
+[2025-11-02 02:45:26,577] [   DEBUG] - local_rank                    : 0
+[2025-11-02 02:45:26,577] [   DEBUG] - log_level                     : -1
+[2025-11-02 02:45:26,577] [   DEBUG] - log_level_replica             : -1
+[2025-11-02 02:45:26,577] [   DEBUG] - log_on_each_node              : True
+[2025-11-02 02:45:26,577] [   DEBUG] - logging_dir                   : tmp_train_mmarco.jsonl/bge-en-icl-paddle/runs/Nov02_02-44-04_b63e119a1648
+[2025-11-02 02:45:26,577] [   DEBUG] - logging_first_step            : False
+[2025-11-02 02:45:26,577] [   DEBUG] - logging_steps                 : 1
+[2025-11-02 02:45:26,577] [   DEBUG] - logging_strategy              : IntervalStrategy.STEPS
+[2025-11-02 02:45:26,577] [   DEBUG] - logical_process_index         : 0
+[2025-11-02 02:45:26,577] [   DEBUG] - lr_end                        : 1e-07
+[2025-11-02 02:45:26,577] [   DEBUG] - lr_scheduler_type             : SchedulerType.LINEAR
+[2025-11-02 02:45:26,577] [   DEBUG] - margin                        : 0.2
+[2025-11-02 02:45:26,578] [   DEBUG] - matryoshka_dims               : [64, 128, 256, 512, 768]
+[2025-11-02 02:45:26,578] [   DEBUG] - matryoshka_loss_weights       : [1, 1, 1, 1, 1]
+[2025-11-02 02:45:26,578] [   DEBUG] - max_evaluate_steps            : -1
+[2025-11-02 02:45:26,578] [   DEBUG] - max_grad_norm                 : 1.0
+[2025-11-02 02:45:26,578] [   DEBUG] - max_steps                     : 100
+[2025-11-02 02:45:26,578] [   DEBUG] - metric_for_best_model         : None
+[2025-11-02 02:45:26,578] [   DEBUG] - metrics_output_path           : None
+[2025-11-02 02:45:26,578] [   DEBUG] - min_lr                        : 0.0
+[2025-11-02 02:45:26,578] [   DEBUG] - minimum_eval_times            : None
+[2025-11-02 02:45:26,578] [   DEBUG] - nccl_comm_group_config        : None
+[2025-11-02 02:45:26,578] [   DEBUG] - negatives_cross_device        : True
+[2025-11-02 02:45:26,578] [   DEBUG] - no_cuda                       : False
+[2025-11-02 02:45:26,578] [   DEBUG] - num_cycles                    : 0.5
+[2025-11-02 02:45:26,578] [   DEBUG] - num_train_epochs              : 1.0
+[2025-11-02 02:45:26,578] [   DEBUG] - offload_optim                 : False
+[2025-11-02 02:45:26,578] [   DEBUG] - optim                         : OptimizerNames.ADAMW
+[2025-11-02 02:45:26,578] [   DEBUG] - optimizer_name_suffix         : shard00
+[2025-11-02 02:45:26,578] [   DEBUG] - ordered_save_group_size       : 0
+[2025-11-02 02:45:26,578] [   DEBUG] - output_dir                    : tmp_train_mmarco.jsonl/bge-en-icl-paddle
+[2025-11-02 02:45:26,579] [   DEBUG] - output_signal_dir             : tmp_train_mmarco.jsonl/bge-en-icl-paddle
+[2025-11-02 02:45:26,579] [   DEBUG] - overwrite_output_dir          : True
+[2025-11-02 02:45:26,579] [   DEBUG] - pad_token_id                  : 0
+[2025-11-02 02:45:26,579] [   DEBUG] - past_index                    : -1
+[2025-11-02 02:45:26,579] [   DEBUG] - pdc_download_ckpt             : False
+[2025-11-02 02:45:26,579] [   DEBUG] - pdc_download_timeout          : 300
+[2025-11-02 02:45:26,579] [   DEBUG] - per_device_eval_batch_size    : 8
+[2025-11-02 02:45:26,579] [   DEBUG] - per_device_train_batch_size   : 1
+[2025-11-02 02:45:26,579] [   DEBUG] - pipeline_parallel_config      : 
+[2025-11-02 02:45:26,579] [   DEBUG] - pipeline_parallel_degree      : 1
+[2025-11-02 02:45:26,579] [   DEBUG] - pipeline_parallel_rank        : 0
+[2025-11-02 02:45:26,579] [   DEBUG] - power                         : 1.0
+[2025-11-02 02:45:26,579] [   DEBUG] - prediction_loss_only          : False
+[2025-11-02 02:45:26,579] [   DEBUG] - process_index                 : 0
+[2025-11-02 02:45:26,579] [   DEBUG] - recompute                     : True
+[2025-11-02 02:45:26,579] [   DEBUG] - refined_recompute             : {}
+[2025-11-02 02:45:26,579] [   DEBUG] - release_grads                 : False
+[2025-11-02 02:45:26,579] [   DEBUG] - remove_unused_columns         : True
+[2025-11-02 02:45:26,579] [   DEBUG] - report_to                     : ['visualdl']
+[2025-11-02 02:45:26,580] [   DEBUG] - resume_from_checkpoint        : None
+[2025-11-02 02:45:26,580] [   DEBUG] - run_name                      : tmp_train_mmarco.jsonl/bge-en-icl-paddle
+[2025-11-02 02:45:26,580] [   DEBUG] - save_on_each_node             : False
+[2025-11-02 02:45:26,580] [   DEBUG] - save_rng_states               : True
+[2025-11-02 02:45:26,580] [   DEBUG] - save_sharded_model            : False
+[2025-11-02 02:45:26,580] [   DEBUG] - save_sharding_stage1_model_include_freeze_params: False
+[2025-11-02 02:45:26,580] [   DEBUG] - save_steps                    : 999999999
+[2025-11-02 02:45:26,580] [   DEBUG] - save_strategy                 : IntervalStrategy.STEPS
+[2025-11-02 02:45:26,580] [   DEBUG] - save_tokenizer                : True
+[2025-11-02 02:45:26,580] [   DEBUG] - save_total_limit              : None
+[2025-11-02 02:45:26,580] [   DEBUG] - scale_loss                    : 32768
+[2025-11-02 02:45:26,580] [   DEBUG] - seed                          : 42
+[2025-11-02 02:45:26,580] [   DEBUG] - sentence_pooling_method       : last
+[2025-11-02 02:45:26,580] [   DEBUG] - sep_parallel_degree           : 1
+[2025-11-02 02:45:26,580] [   DEBUG] - sequence_parallel             : False
+[2025-11-02 02:45:26,580] [   DEBUG] - sequence_parallel_config      : 
+[2025-11-02 02:45:26,580] [   DEBUG] - sharding                      : [<ShardingOption.FULL_SHARD: 'stage3'>, <ShardingOption.OFFLOAD: 'offload'>]
+[2025-11-02 02:45:26,580] [   DEBUG] - sharding_comm_buffer_size_MB  : -1
+[2025-11-02 02:45:26,580] [   DEBUG] - sharding_degree               : -1
+[2025-11-02 02:45:26,580] [   DEBUG] - sharding_parallel_config      : 
+[2025-11-02 02:45:26,581] [   DEBUG] - sharding_parallel_degree      : 4
+[2025-11-02 02:45:26,581] [   DEBUG] - sharding_parallel_mesh_dimension: dp
+[2025-11-02 02:45:26,581] [   DEBUG] - sharding_parallel_rank        : 0
+[2025-11-02 02:45:26,581] [   DEBUG] - should_load_dataset           : True
+[2025-11-02 02:45:26,581] [   DEBUG] - should_load_sharding_stage1_model: False
+[2025-11-02 02:45:26,581] [   DEBUG] - should_log                    : True
+[2025-11-02 02:45:26,581] [   DEBUG] - should_save                   : True
+[2025-11-02 02:45:26,581] [   DEBUG] - should_save_model_state       : True
+[2025-11-02 02:45:26,581] [   DEBUG] - should_save_model_with_tensor_fusion: False
+[2025-11-02 02:45:26,581] [   DEBUG] - should_save_sharding_stage1_model: False
+[2025-11-02 02:45:26,603] [   DEBUG] - skip_data_intervals           : None
+[2025-11-02 02:45:26,603] [   DEBUG] - skip_memory_metrics           : True
+[2025-11-02 02:45:26,603] [   DEBUG] - skip_profile_timer            : True
+[2025-11-02 02:45:26,603] [   DEBUG] - split_inputs_sequence_dim     : True
+[2025-11-02 02:45:26,603] [   DEBUG] - split_norm_comm               : False
+[2025-11-02 02:45:26,603] [   DEBUG] - temperature                   : 0.01
+[2025-11-02 02:45:26,603] [   DEBUG] - tensor_parallel_config        : 
+[2025-11-02 02:45:26,603] [   DEBUG] - tensor_parallel_degree        : 1
+[2025-11-02 02:45:26,603] [   DEBUG] - tensor_parallel_rank          : 0
+[2025-11-02 02:45:26,603] [   DEBUG] - tensorwise_offload_optimizer  : False
+[2025-11-02 02:45:26,603] [   DEBUG] - to_static                     : False
+[2025-11-02 02:45:26,603] [   DEBUG] - train_batch_size              : 1
+[2025-11-02 02:45:26,603] [   DEBUG] - unified_checkpoint            : False
+[2025-11-02 02:45:26,603] [   DEBUG] - unified_checkpoint_config     : 
+[2025-11-02 02:45:26,603] [   DEBUG] - use_async_save                : False
+[2025-11-02 02:45:26,604] [   DEBUG] - use_expert_parallel           : False
+[2025-11-02 02:45:26,604] [   DEBUG] - use_hybrid_parallel           : True
+[2025-11-02 02:45:26,604] [   DEBUG] - use_inbatch_neg               : False
+[2025-11-02 02:45:26,604] [   DEBUG] - use_lowprecision_moment       : False
+[2025-11-02 02:45:26,604] [   DEBUG] - use_matryoshka                : False
+[2025-11-02 02:45:26,604] [   DEBUG] - wandb_api_key                 : None
+[2025-11-02 02:45:26,604] [   DEBUG] - wandb_http_proxy              : None
+[2025-11-02 02:45:26,604] [   DEBUG] - warmup_ratio                  : 0.0
+[2025-11-02 02:45:26,604] [   DEBUG] - warmup_steps                  : 10
+[2025-11-02 02:45:26,604] [   DEBUG] - weight_decay                  : 0.0
+[2025-11-02 02:45:26,604] [   DEBUG] - weight_name_suffix            : 
+[2025-11-02 02:45:26,604] [   DEBUG] - world_size                    : 4
+[2025-11-02 02:45:26,604] [   DEBUG] - zcc_ema_interval              : 1
+[2025-11-02 02:45:26,604] [   DEBUG] - zcc_pipeline_hooks_capacity_usage: 0.6
+[2025-11-02 02:45:26,604] [   DEBUG] - zcc_save_ema_coef             : None
+[2025-11-02 02:45:26,604] [   DEBUG] - zcc_workers_num               : 3
+[2025-11-02 02:45:26,604] [   DEBUG] - 
+[2025-11-02 02:45:26,604] [    INFO] - Starting training from resume_from_checkpoint : None
+W1102 02:45:26.762852 76025 nccl_comm_context.cc:70] ncclCommInitRankConfigMemOpt is not supported.
+[2025-11-02 02:45:27,402] [ WARNING] group_sharded.py:148 - the input of scaler is None, please ensure the logic of your scaler outside is same as GroupShardedScaler.
+[2025-11-02 02:45:27,404] [ WARNING] group_sharded_stage3.py:195 - While using ClipGradByGlobalNorm in GroupShardedStage3, the grad clip of original optimizer will be changed.
+W1102 02:45:27.407776 76025 nccl_comm_context.cc:70] ncclCommInitRankConfigMemOpt is not supported.
+[2025-11-02 02:46:01,955] [    INFO] - [timelog] checkpoint loading time: 0.00s (2025-11-02 02:46:01) 
+[2025-11-02 02:46:01,955] [    INFO] - ***** Running training *****
+[2025-11-02 02:46:01,955] [    INFO] -   Num examples = 200,001
+[2025-11-02 02:46:01,955] [    INFO] -   Num Epochs = 1
+[2025-11-02 02:46:01,991] [    INFO] -   Instantaneous batch size per device = 1
+[2025-11-02 02:46:01,991] [    INFO] -   Total train batch size (w. parallel, distributed & accumulation) = 128
+[2025-11-02 02:46:01,992] [    INFO] -   Gradient Accumulation steps = 32
+[2025-11-02 02:46:02,005] [    INFO] -   Total optimization steps = 100
+[2025-11-02 02:46:02,005] [    INFO] -   Total num train samples = 12,800
+[2025-11-02 02:46:02,008] [   DEBUG] -   Number of trainable parameters = 18,874,368 (per device)
+
+TrainProcess:   0%|          | 0/100 [00:00<?, ?it/s]W1102 02:46:15.102584 76025 multiply_fwd_func.cc:82] got different data type, run type promotion automatically, this may cause data type been changed.
+W1102 02:46:27.257807 76025 dygraph_functions.cc:94044] got different data type, run type promotion automatically, this may cause data type been changed.
+
+TrainProcess:   1%|          | 1/100 [05:25<8:57:33, 325.79s/it][2025-11-02 02:51:28,012] [    INFO] - loss: 0.66553652, learning_rate: 1e-05, global_step: 1, current_memory_allocated: 13.53714394569397, current_memory_reserved: 71.03532481193542, max_memory_allocated: 58.337307929992676, max_memory_reserved: 71.03532481193542, interval_runtime: 325.8161, interval_samples_per_second: 0.3929, interval_steps_per_second: 0.0031, progress_or_epoch: 0.0006
+LAUNCH WARNING 2025-11-02 02:54:29,656 save gpu log failed: Command '['nvidia-smi', '--format=csv,noheader,nounits', '--query-gpu=index,utilization.gpu,memory.total,memory.used,memory.free,timestamp', '--id=0,1,2,3']' timed out after 3 seconds
+
+TrainProcess:   2%|▏         | 2/100 [11:07<9:07:04, 334.95s/it][2025-11-02 02:57:09,275] [    INFO] - loss: 0.79860699, learning_rate: 2e-05, global_step: 2, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.3518, interval_samples_per_second: 0.375, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0013
+
+TrainProcess:   3%|▎         | 3/100 [16:50<9:07:36, 338.73s/it][2025-11-02 03:02:52,473] [    INFO] - loss: 0.75874907, learning_rate: 3e-05, global_step: 3, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.2247, interval_samples_per_second: 0.3729, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0019
+
+TrainProcess:   4%|▍         | 4/100 [22:31<9:03:24, 339.63s/it][2025-11-02 03:08:33,539] [    INFO] - loss: 0.66077542, learning_rate: 4e-05, global_step: 4, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.0198, interval_samples_per_second: 0.3753, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0026
+
+TrainProcess:   5%|▌         | 5/100 [28:13<8:59:09, 340.52s/it][2025-11-02 03:14:15,581] [    INFO] - loss: 0.54946154, learning_rate: 5e-05, global_step: 5, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.096, interval_samples_per_second: 0.3742, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0032
+
+TrainProcess:   6%|▌         | 6/100 [33:56<8:54:53, 341.42s/it][2025-11-02 03:19:58,776] [    INFO] - loss: 0.50007021, learning_rate: 6e-05, global_step: 6, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.1689, interval_samples_per_second: 0.373, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0038
+
+TrainProcess:   7%|▋         | 7/100 [39:38<8:49:36, 341.69s/it][2025-11-02 03:25:40,987] [    INFO] - loss: 0.36123151, learning_rate: 7e-05, global_step: 7, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.2283, interval_samples_per_second: 0.374, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0045
+
+TrainProcess:   8%|▊         | 8/100 [45:22<8:44:52, 342.31s/it][2025-11-02 03:31:24,677] [    INFO] - loss: 0.31946236, learning_rate: 8e-05, global_step: 8, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.6618, interval_samples_per_second: 0.3725, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0051
+
+TrainProcess:   9%|▉         | 9/100 [51:04<8:38:59, 342.19s/it][2025-11-02 03:37:06,587] [    INFO] - loss: 0.36731112, learning_rate: 9e-05, global_step: 9, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.9238, interval_samples_per_second: 0.3744, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0058
+
+TrainProcess:  10%|█         | 10/100 [56:47<8:33:38, 342.42s/it][2025-11-02 03:42:49,547] [    INFO] - loss: 0.21933824, learning_rate: 0.0001, global_step: 10, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.9429, interval_samples_per_second: 0.3732, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0064
+
+TrainProcess:  11%|█         | 11/100 [1:02:30<8:28:05, 342.54s/it][2025-11-02 03:48:32,329] [    INFO] - loss: 0.23952383, learning_rate: 9.889e-05, global_step: 11, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.7925, interval_samples_per_second: 0.3734, interval_steps_per_second: 0.0029, progress_or_epoch: 0.007
+
+TrainProcess:  12%|█▏        | 12/100 [1:08:12<8:22:13, 342.43s/it][2025-11-02 03:54:14,498] [    INFO] - loss: 0.33754838, learning_rate: 9.778e-05, global_step: 12, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.1884, interval_samples_per_second: 0.3741, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0077
+
+TrainProcess:  13%|█▎        | 13/100 [1:13:53<8:16:05, 342.13s/it][2025-11-02 03:59:56,001] [    INFO] - loss: 0.23905057, learning_rate: 9.667e-05, global_step: 13, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.438, interval_samples_per_second: 0.3749, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0083
+
+TrainProcess:  14%|█▍        | 14/100 [1:19:34<8:09:48, 341.73s/it][2025-11-02 04:05:36,763] [    INFO] - loss: 0.15280873, learning_rate: 9.556e-05, global_step: 14, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 340.8037, interval_samples_per_second: 0.3756, interval_steps_per_second: 0.0029, progress_or_epoch: 0.009
+
+TrainProcess:  15%|█▌        | 15/100 [1:25:17<8:04:24, 341.94s/it][2025-11-02 04:11:19,178] [    INFO] - loss: 0.1158784, learning_rate: 9.444e-05, global_step: 15, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.427, interval_samples_per_second: 0.3738, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0096
+
+TrainProcess:  16%|█▌        | 16/100 [1:30:58<7:58:21, 341.69s/it][2025-11-02 04:17:00,331] [    INFO] - loss: 0.20819521, learning_rate: 9.333e-05, global_step: 16, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.1088, interval_samples_per_second: 0.3752, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0102
+
+TrainProcess:  17%|█▋        | 17/100 [1:36:38<7:51:59, 341.20s/it][2025-11-02 04:22:40,408] [    INFO] - loss: 0.18641633, learning_rate: 9.222e-05, global_step: 17, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 340.0633, interval_samples_per_second: 0.3764, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0109
+
+TrainProcess:  18%|█▊        | 18/100 [1:42:20<7:46:47, 341.56s/it][2025-11-02 04:28:22,741] [    INFO] - loss: 0.43880731, learning_rate: 9.111e-05, global_step: 18, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.3884, interval_samples_per_second: 0.3738, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0115
+
+TrainProcess:  19%|█▉        | 19/100 [1:48:02<7:41:17, 341.70s/it][2025-11-02 04:34:04,762] [    INFO] - loss: 0.21998173, learning_rate: 9e-05, global_step: 19, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.0259, interval_samples_per_second: 0.3742, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0122
+
+TrainProcess:  20%|██        | 20/100 [1:53:46<7:36:20, 342.26s/it][2025-11-02 04:39:48,329] [    INFO] - loss: 0.15043002, learning_rate: 8.889e-05, global_step: 20, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.5673, interval_samples_per_second: 0.3726, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0128
+
+TrainProcess:  21%|██        | 21/100 [1:59:27<7:30:15, 341.97s/it][2025-11-02 04:45:29,639] [    INFO] - loss: 0.23122263, learning_rate: 8.778e-05, global_step: 21, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.3086, interval_samples_per_second: 0.375, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0134
+
+TrainProcess:  22%|██▏       | 22/100 [2:05:06<7:23:28, 341.13s/it][2025-11-02 04:51:08,819] [    INFO] - loss: 0.14337724, learning_rate: 8.667e-05, global_step: 22, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.1615, interval_samples_per_second: 0.3774, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0141
+
+TrainProcess:  23%|██▎       | 23/100 [2:10:45<7:16:56, 340.48s/it][2025-11-02 04:56:47,822] [    INFO] - loss: 0.17516977, learning_rate: 8.556e-05, global_step: 23, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 338.9538, interval_samples_per_second: 0.3776, interval_steps_per_second: 0.003, progress_or_epoch: 0.0147
+
+TrainProcess:  24%|██▍       | 24/100 [2:16:27<7:11:37, 340.76s/it][2025-11-02 05:02:29,222] [    INFO] - loss: 0.19867271, learning_rate: 8.444e-05, global_step: 24, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.4281, interval_samples_per_second: 0.3749, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0154
+
+TrainProcess:  25%|██▌       | 25/100 [2:22:11<7:07:15, 341.80s/it][2025-11-02 05:08:13,447] [    INFO] - loss: 0.26363957, learning_rate: 8.333e-05, global_step: 25, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 344.2292, interval_samples_per_second: 0.3718, interval_steps_per_second: 0.0029, progress_or_epoch: 0.016
+
+TrainProcess:  26%|██▌       | 26/100 [2:27:54<7:02:14, 342.36s/it][2025-11-02 05:13:57,159] [    INFO] - loss: 0.19190687, learning_rate: 8.222e-05, global_step: 26, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.6564, interval_samples_per_second: 0.3725, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0166
+
+TrainProcess:  27%|██▋       | 27/100 [2:33:38<6:57:04, 342.80s/it][2025-11-02 05:19:40,885] [    INFO] - loss: 0.07733226, learning_rate: 8.111e-05, global_step: 27, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.8287, interval_samples_per_second: 0.3723, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0173
+
+TrainProcess:  28%|██▊       | 28/100 [2:39:22<6:51:32, 342.96s/it][2025-11-02 05:25:24,212] [    INFO] - loss: 0.16453004, learning_rate: 8e-05, global_step: 28, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.3245, interval_samples_per_second: 0.3728, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0179
+
+TrainProcess:  29%|██▉       | 29/100 [2:45:04<6:45:41, 342.84s/it][2025-11-02 05:31:06,908] [    INFO] - loss: 0.19541007, learning_rate: 7.889e-05, global_step: 29, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.571, interval_samples_per_second: 0.3736, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0186
+
+TrainProcess:  30%|███       | 30/100 [2:50:46<6:39:45, 342.64s/it][2025-11-02 05:36:48,993] [    INFO] - loss: 0.33096033, learning_rate: 7.778e-05, global_step: 30, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.1806, interval_samples_per_second: 0.3741, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0192
+
+TrainProcess:  31%|███       | 31/100 [2:56:27<6:33:27, 342.14s/it][2025-11-02 05:42:29,968] [    INFO] - loss: 0.12353015, learning_rate: 7.667e-05, global_step: 31, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 340.9621, interval_samples_per_second: 0.3754, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0198
+
+TrainProcess:  32%|███▏      | 32/100 [3:02:11<6:28:16, 342.59s/it][2025-11-02 05:48:13,575] [    INFO] - loss: 0.12434918, learning_rate: 7.556e-05, global_step: 32, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.6413, interval_samples_per_second: 0.3725, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0205
+
+TrainProcess:  33%|███▎      | 33/100 [3:07:53<6:22:29, 342.54s/it][2025-11-02 05:53:56,041] [    INFO] - loss: 0.25178021, learning_rate: 7.444e-05, global_step: 33, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.4095, interval_samples_per_second: 0.3738, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0211
+
+TrainProcess:  34%|███▍      | 34/100 [3:13:37<6:17:16, 342.98s/it][2025-11-02 05:59:40,059] [    INFO] - loss: 0.18408251, learning_rate: 7.333e-05, global_step: 34, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 344.0249, interval_samples_per_second: 0.3721, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0218
+
+TrainProcess:  35%|███▌      | 35/100 [3:19:24<6:12:36, 343.95s/it][2025-11-02 06:05:26,204] [    INFO] - loss: 0.18803912, learning_rate: 7.222e-05, global_step: 35, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 346.2082, interval_samples_per_second: 0.3697, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0224
+
+TrainProcess:  36%|███▌      | 36/100 [3:25:06<6:06:14, 343.35s/it][2025-11-02 06:11:08,193] [    INFO] - loss: 0.09699774, learning_rate: 7.111e-05, global_step: 36, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.957, interval_samples_per_second: 0.3743, interval_steps_per_second: 0.0029, progress_or_epoch: 0.023
+
+TrainProcess:  37%|███▋      | 37/100 [3:30:48<6:00:15, 343.11s/it][2025-11-02 06:16:50,700] [    INFO] - loss: 0.22247481, learning_rate: 7e-05, global_step: 37, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.5402, interval_samples_per_second: 0.3737, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0237
+
+TrainProcess:  38%|███▊      | 38/100 [3:36:32<5:54:55, 343.47s/it][2025-11-02 06:22:35,039] [    INFO] - loss: 0.15630615, learning_rate: 6.889e-05, global_step: 38, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 344.3181, interval_samples_per_second: 0.3717, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0243
+
+TrainProcess:  39%|███▉      | 39/100 [3:42:12<5:47:58, 342.26s/it][2025-11-02 06:28:14,493] [    INFO] - loss: 0.14717603, learning_rate: 6.778e-05, global_step: 39, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.4414, interval_samples_per_second: 0.3771, interval_steps_per_second: 0.0029, progress_or_epoch: 0.025
+
+TrainProcess:  40%|████      | 40/100 [3:47:54<5:42:05, 342.10s/it][2025-11-02 06:33:56,254] [    INFO] - loss: 0.18778747, learning_rate: 6.667e-05, global_step: 40, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.711, interval_samples_per_second: 0.3746, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0256
+
+TrainProcess:  41%|████      | 41/100 [3:53:38<5:37:10, 342.88s/it][2025-11-02 06:39:40,938] [    INFO] - loss: 0.22823358, learning_rate: 6.556e-05, global_step: 41, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 344.7161, interval_samples_per_second: 0.3713, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0262
+
+TrainProcess:  42%|████▏     | 42/100 [3:59:20<5:31:12, 342.63s/it][2025-11-02 06:45:22,938] [    INFO] - loss: 0.11471742, learning_rate: 6.444e-05, global_step: 42, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.042, interval_samples_per_second: 0.3742, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0269
+
+TrainProcess:  43%|████▎     | 43/100 [4:05:04<5:25:52, 343.02s/it][2025-11-02 06:51:06,865] [    INFO] - loss: 0.12735146, learning_rate: 6.333e-05, global_step: 43, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.9219, interval_samples_per_second: 0.3722, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0275
+
+TrainProcess:  44%|████▍     | 44/100 [4:10:46<5:19:47, 342.63s/it][2025-11-02 06:56:48,606] [    INFO] - loss: 0.2830233, learning_rate: 6.222e-05, global_step: 44, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.7313, interval_samples_per_second: 0.3746, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0282
+
+TrainProcess:  45%|████▌     | 45/100 [4:16:30<5:14:23, 342.97s/it][2025-11-02 07:02:32,410] [    INFO] - loss: 0.13983661, learning_rate: 6.111e-05, global_step: 45, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.7639, interval_samples_per_second: 0.3723, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0288
+
+TrainProcess:  46%|████▌     | 46/100 [4:22:13<5:08:45, 343.06s/it][2025-11-02 07:08:15,725] [    INFO] - loss: 0.2090162, learning_rate: 6e-05, global_step: 46, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.2787, interval_samples_per_second: 0.3729, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0294
+
+TrainProcess:  47%|████▋     | 47/100 [4:27:56<5:03:01, 343.05s/it][2025-11-02 07:13:58,670] [    INFO] - loss: 0.19593245, learning_rate: 5.889e-05, global_step: 47, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.0085, interval_samples_per_second: 0.3732, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0301
+
+TrainProcess:  48%|████▊     | 48/100 [4:33:39<4:57:13, 342.95s/it][2025-11-02 07:19:41,350] [    INFO] - loss: 0.1040743, learning_rate: 5.778e-05, global_step: 48, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.7244, interval_samples_per_second: 0.3735, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0307
+
+TrainProcess:  49%|████▉     | 49/100 [4:39:20<4:51:03, 342.42s/it][2025-11-02 07:25:22,585] [    INFO] - loss: 0.09637725, learning_rate: 5.667e-05, global_step: 49, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.1844, interval_samples_per_second: 0.3752, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0314
+
+TrainProcess:  50%|█████     | 50/100 [4:45:04<4:45:46, 342.93s/it][2025-11-02 07:31:06,698] [    INFO] - loss: 0.1762166, learning_rate: 5.556e-05, global_step: 50, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 344.1284, interval_samples_per_second: 0.372, interval_steps_per_second: 0.0029, progress_or_epoch: 0.032
+
+TrainProcess:  51%|█████     | 51/100 [4:50:46<4:39:43, 342.52s/it][2025-11-02 07:36:48,312] [    INFO] - loss: 0.21373814, learning_rate: 5.444e-05, global_step: 51, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.5447, interval_samples_per_second: 0.3748, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0326
+
+TrainProcess:  52%|█████▏    | 52/100 [4:56:27<4:33:50, 342.29s/it][2025-11-02 07:42:30,010] [    INFO] - loss: 0.15427035, learning_rate: 5.333e-05, global_step: 52, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.7748, interval_samples_per_second: 0.3745, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0333
+
+TrainProcess:  53%|█████▎    | 53/100 [5:02:11<4:28:24, 342.66s/it][2025-11-02 07:48:13,578] [    INFO] - loss: 0.06576741, learning_rate: 5.222e-05, global_step: 53, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.5083, interval_samples_per_second: 0.3726, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0339
+
+TrainProcess:  54%|█████▍    | 54/100 [5:07:54<4:22:52, 342.89s/it][2025-11-02 07:53:56,924] [    INFO] - loss: 0.15179139, learning_rate: 5.111e-05, global_step: 54, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.4253, interval_samples_per_second: 0.3727, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0346
+
+TrainProcess:  55%|█████▌    | 55/100 [5:13:40<4:17:45, 343.68s/it][2025-11-02 07:59:42,524] [    INFO] - loss: 0.18471116, learning_rate: 5e-05, global_step: 55, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 345.5406, interval_samples_per_second: 0.3704, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0352
+
+TrainProcess:  56%|█████▌    | 56/100 [5:19:21<4:11:25, 342.85s/it][2025-11-02 08:05:23,429] [    INFO] - loss: 0.27741587, learning_rate: 4.889e-05, global_step: 56, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 340.8951, interval_samples_per_second: 0.3755, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0358
+
+TrainProcess:  57%|█████▋    | 57/100 [5:25:02<4:05:22, 342.39s/it][2025-11-02 08:11:04,778] [    INFO] - loss: 0.14725566, learning_rate: 4.778e-05, global_step: 57, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.3318, interval_samples_per_second: 0.375, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0365
+
+TrainProcess:  58%|█████▊    | 58/100 [5:30:43<3:59:17, 341.84s/it][2025-11-02 08:16:45,300] [    INFO] - loss: 0.07949138, learning_rate: 4.667e-05, global_step: 58, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 340.5538, interval_samples_per_second: 0.3759, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0371
+
+TrainProcess:  59%|█████▉    | 59/100 [5:36:22<3:53:07, 341.16s/it][2025-11-02 08:22:24,948] [    INFO] - loss: 0.07810926, learning_rate: 4.556e-05, global_step: 59, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.5757, interval_samples_per_second: 0.3769, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0378
+
+TrainProcess:  60%|██████    | 60/100 [5:42:02<3:47:11, 340.79s/it][2025-11-02 08:28:04,809] [    INFO] - loss: 0.16819191, learning_rate: 4.444e-05, global_step: 60, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.917, interval_samples_per_second: 0.3766, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0384
+
+TrainProcess:  61%|██████    | 61/100 [5:47:43<3:41:26, 340.69s/it][2025-11-02 08:33:45,253] [    INFO] - loss: 0.06770384, learning_rate: 4.333e-05, global_step: 61, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 340.465, interval_samples_per_second: 0.376, interval_steps_per_second: 0.0029, progress_or_epoch: 0.039
+
+TrainProcess:  62%|██████▏   | 62/100 [5:53:24<3:35:57, 340.97s/it][2025-11-02 08:39:26,850] [    INFO] - loss: 0.13749784, learning_rate: 4.222e-05, global_step: 62, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.6372, interval_samples_per_second: 0.3747, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0397
+
+TrainProcess:  63%|██████▎   | 63/100 [5:59:04<3:29:57, 340.48s/it][2025-11-02 08:45:06,173] [    INFO] - loss: 0.0682075, learning_rate: 4.111e-05, global_step: 63, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.3272, interval_samples_per_second: 0.3772, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0403
+
+TrainProcess:  64%|██████▍   | 64/100 [6:04:44<3:24:13, 340.39s/it][2025-11-02 08:50:46,413] [    INFO] - loss: 0.1639846, learning_rate: 4e-05, global_step: 64, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 340.1814, interval_samples_per_second: 0.3763, interval_steps_per_second: 0.0029, progress_or_epoch: 0.041
+
+TrainProcess:  65%|██████▌   | 65/100 [6:10:25<3:18:40, 340.57s/it][2025-11-02 08:56:27,337] [    INFO] - loss: 0.10053152, learning_rate: 3.889e-05, global_step: 65, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 340.9927, interval_samples_per_second: 0.3754, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0416
+
+TrainProcess:  66%|██████▌   | 66/100 [6:16:07<3:13:21, 341.21s/it][2025-11-02 09:02:10,095] [    INFO] - loss: 0.20406103, learning_rate: 3.778e-05, global_step: 66, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.721, interval_samples_per_second: 0.3735, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0422
+
+TrainProcess:  67%|██████▋   | 67/100 [6:21:52<3:08:10, 342.15s/it][2025-11-02 09:07:54,401] [    INFO] - loss: 0.31804168, learning_rate: 3.667e-05, global_step: 67, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 344.3248, interval_samples_per_second: 0.3717, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0429
+
+TrainProcess:  68%|██████▊   | 68/100 [6:27:35<3:02:35, 342.36s/it][2025-11-02 09:13:37,341] [    INFO] - loss: 0.15694392, learning_rate: 3.556e-05, global_step: 68, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.8423, interval_samples_per_second: 0.3733, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0435
+
+TrainProcess:  69%|██████▉   | 69/100 [6:33:17<2:56:54, 342.40s/it][2025-11-02 09:19:19,712] [    INFO] - loss: 0.25463253, learning_rate: 3.444e-05, global_step: 69, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.481, interval_samples_per_second: 0.3737, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0442
+
+TrainProcess:  70%|███████   | 70/100 [6:38:57<2:50:48, 341.62s/it][2025-11-02 09:24:59,569] [    INFO] - loss: 0.08691931, learning_rate: 3.333e-05, global_step: 70, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.8227, interval_samples_per_second: 0.3767, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0448
+
+TrainProcess:  71%|███████   | 71/100 [6:44:39<2:45:08, 341.67s/it][2025-11-02 09:30:41,296] [    INFO] - loss: 0.10098779, learning_rate: 3.222e-05, global_step: 71, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.7801, interval_samples_per_second: 0.3745, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0454
+
+TrainProcess:  72%|███████▏  | 72/100 [6:50:22<2:39:39, 342.14s/it][2025-11-02 09:36:24,557] [    INFO] - loss: 0.28246444, learning_rate: 3.111e-05, global_step: 72, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.2408, interval_samples_per_second: 0.3729, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0461
+
+TrainProcess:  73%|███████▎  | 73/100 [6:56:06<2:34:11, 342.63s/it][2025-11-02 09:42:08,489] [    INFO] - loss: 0.1267423, learning_rate: 3e-05, global_step: 73, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.7821, interval_samples_per_second: 0.3723, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0467
+
+TrainProcess:  74%|███████��  | 74/100 [7:01:47<2:28:19, 342.28s/it][2025-11-02 09:47:49,871] [    INFO] - loss: 0.10064691, learning_rate: 2.889e-05, global_step: 74, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.4666, interval_samples_per_second: 0.3749, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0474
+
+TrainProcess:  75%|███████▌  | 75/100 [7:07:29<2:22:32, 342.12s/it][2025-11-02 09:53:31,531] [    INFO] - loss: 0.17008591, learning_rate: 2.778e-05, global_step: 75, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.7372, interval_samples_per_second: 0.3746, interval_steps_per_second: 0.0029, progress_or_epoch: 0.048
+
+TrainProcess:  76%|███████▌  | 76/100 [7:13:11<2:16:53, 342.22s/it][2025-11-02 09:59:14,031] [    INFO] - loss: 0.18092716, learning_rate: 2.667e-05, global_step: 76, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.4487, interval_samples_per_second: 0.3738, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0486
+
+TrainProcess:  77%|███████▋  | 77/100 [7:18:55<2:11:19, 342.59s/it][2025-11-02 10:04:57,453] [    INFO] - loss: 0.12117821, learning_rate: 2.556e-05, global_step: 77, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.4522, interval_samples_per_second: 0.3727, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0493
+
+TrainProcess:  78%|███████▊  | 78/100 [7:24:38<2:05:39, 342.71s/it][2025-11-02 10:10:40,460] [    INFO] - loss: 0.05095053, learning_rate: 2.444e-05, global_step: 78, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.9898, interval_samples_per_second: 0.3732, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0499
+
+TrainProcess:  79%|███████▉  | 79/100 [7:30:22<2:00:07, 343.19s/it][2025-11-02 10:16:24,796] [    INFO] - loss: 0.34499228, learning_rate: 2.333e-05, global_step: 79, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 344.3267, interval_samples_per_second: 0.3717, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0506
+
+TrainProcess:  80%|████████  | 80/100 [7:36:04<1:54:13, 342.65s/it][2025-11-02 10:22:06,211] [    INFO] - loss: 0.28233987, learning_rate: 2.222e-05, global_step: 80, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.3883, interval_samples_per_second: 0.3749, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0512
+
+TrainProcess:  81%|████████  | 81/100 [7:41:46<1:48:31, 342.69s/it][2025-11-02 10:27:48,939] [    INFO] - loss: 0.10262841, learning_rate: 2.111e-05, global_step: 81, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.7899, interval_samples_per_second: 0.3734, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0518
+
+TrainProcess:  82%|████████▏ | 82/100 [7:47:28<1:42:44, 342.45s/it][2025-11-02 10:33:30,831] [    INFO] - loss: 0.21901381, learning_rate: 2e-05, global_step: 82, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.8783, interval_samples_per_second: 0.3744, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0525
+
+TrainProcess:  83%|████████▎ | 83/100 [7:53:11<1:37:03, 342.59s/it][2025-11-02 10:39:13,774] [    INFO] - loss: 0.06807154, learning_rate: 1.889e-05, global_step: 83, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.9045, interval_samples_per_second: 0.3733, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0531
+
+TrainProcess:  84%|████████▍ | 84/100 [7:58:54<1:31:21, 342.61s/it][2025-11-02 10:44:56,424] [    INFO] - loss: 0.1241712, learning_rate: 1.778e-05, global_step: 84, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.6813, interval_samples_per_second: 0.3735, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0538
+
+TrainProcess:  85%|████████▌ | 85/100 [8:04:37<1:25:42, 342.81s/it][2025-11-02 10:50:39,713] [    INFO] - loss: 0.17290878, learning_rate: 1.667e-05, global_step: 85, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.2761, interval_samples_per_second: 0.3729, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0544
+
+TrainProcess:  86%|████████▌ | 86/100 [8:10:23<1:20:11, 343.65s/it][2025-11-02 10:56:25,353] [    INFO] - loss: 0.15047735, learning_rate: 1.556e-05, global_step: 86, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 345.6114, interval_samples_per_second: 0.3704, interval_steps_per_second: 0.0029, progress_or_epoch: 0.055
+
+TrainProcess:  87%|████████▋ | 87/100 [8:16:07<1:14:28, 343.73s/it][2025-11-02 11:02:09,210] [    INFO] - loss: 0.16767275, learning_rate: 1.444e-05, global_step: 87, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.8996, interval_samples_per_second: 0.3722, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0557
+
+TrainProcess:  88%|████████▊ | 88/100 [8:21:50<1:08:44, 343.69s/it][2025-11-02 11:07:52,901] [    INFO] - loss: 0.12810075, learning_rate: 1.333e-05, global_step: 88, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 343.6097, interval_samples_per_second: 0.3725, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0563
+
+TrainProcess:  89%|████████▉ | 89/100 [8:27:33<1:02:57, 343.39s/it][2025-11-02 11:13:35,506] [    INFO] - loss: 0.16182154, learning_rate: 1.222e-05, global_step: 89, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 342.6884, interval_samples_per_second: 0.3735, interval_steps_per_second: 0.0029, progress_or_epoch: 0.057
+
+TrainProcess:  90%|█████████ | 90/100 [8:33:18<57:18, 343.87s/it]  [2025-11-02 11:19:20,544] [    INFO] - loss: 0.21699762, learning_rate: 1.111e-05, global_step: 90, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 344.9714, interval_samples_per_second: 0.371, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0576
+
+TrainProcess:  91%|█████████ | 91/100 [8:38:59<51:27, 343.07s/it][2025-11-02 11:25:01,695] [    INFO] - loss: 0.22559232, learning_rate: 1e-05, global_step: 91, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 341.2129, interval_samples_per_second: 0.3751, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0582
+
+TrainProcess:  92%|█████████▏| 92/100 [8:44:39<45:36, 342.08s/it][2025-11-02 11:30:41,462] [    INFO] - loss: 0.21424091, learning_rate: 8.889e-06, global_step: 92, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.7834, interval_samples_per_second: 0.3767, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0589
+
+TrainProcess:  93%|█████████▎| 93/100 [8:50:18<39:47, 341.12s/it][2025-11-02 11:36:20,303] [    INFO] - loss: 0.17718983, learning_rate: 7.778e-06, global_step: 93, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 338.872, interval_samples_per_second: 0.3777, interval_steps_per_second: 0.003, progress_or_epoch: 0.0595
+
+TrainProcess:  94%|█████████▍| 94/100 [8:55:56<34:01, 340.31s/it][2025-11-02 11:41:58,884] [    INFO] - loss: 0.17421329, learning_rate: 6.667e-06, global_step: 94, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 338.4266, interval_samples_per_second: 0.3782, interval_steps_per_second: 0.003, progress_or_epoch: 0.0602
+
+TrainProcess:  95%|█████████▌| 95/100 [9:01:36<28:21, 340.20s/it][2025-11-02 11:47:38,743] [    INFO] - loss: 0.09569776, learning_rate: 5.556e-06, global_step: 95, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.9515, interval_samples_per_second: 0.3765, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0608
+
+TrainProcess:  96%|█████████▌| 96/100 [9:07:14<22:38, 339.59s/it][2025-11-02 11:53:16,886] [    INFO] - loss: 0.13749444, learning_rate: 4.444e-06, global_step: 96, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 338.1433, interval_samples_per_second: 0.3785, interval_steps_per_second: 0.003, progress_or_epoch: 0.0614
+
+TrainProcess:  97%|█████████▋| 97/100 [9:12:52<16:57, 339.15s/it][2025-11-02 11:58:55,034] [    INFO] - loss: 0.14223635, learning_rate: 3.333e-06, global_step: 97, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 338.1388, interval_samples_per_second: 0.3785, interval_steps_per_second: 0.003, progress_or_epoch: 0.0621
+
+TrainProcess:  98%|█████████▊| 98/100 [9:18:32<11:18, 339.21s/it][2025-11-02 12:04:34,501] [    INFO] - loss: 0.16814822, learning_rate: 2.222e-06, global_step: 98, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.3339, interval_samples_per_second: 0.3772, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0627
+
+TrainProcess:  99%|█████████▉| 99/100 [9:24:09<05:38, 338.71s/it][2025-11-02 12:10:11,891] [    INFO] - loss: 0.24756187, learning_rate: 1.111e-06, global_step: 99, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 337.5474, interval_samples_per_second: 0.3792, interval_steps_per_second: 0.003, progress_or_epoch: 0.0634
+
+TrainProcess: 100%|██████████| 100/100 [9:29:49<00:00, 339.05s/it][2025-11-02 12:15:51,774] [    INFO] - loss: 0.17651314, learning_rate: 0.0, global_step: 100, current_memory_allocated: 13.53714370727539, current_memory_reserved: 71.03532481193542, max_memory_allocated: 59.20164966583252, max_memory_reserved: 71.03532481193542, interval_runtime: 339.8511, interval_samples_per_second: 0.3766, interval_steps_per_second: 0.0029, progress_or_epoch: 0.064
+[2025-11-02 12:15:51,775] [    INFO] - 
+Training completed. 
+
+[2025-11-02 12:15:51,784] [    INFO] - train_runtime: 34189.7588, train_samples_per_second: 0.3744, train_steps_per_second: 0.0029, train_loss: 0.20930041134357452, progress_or_epoch: 0.064
+
+TrainProcess: 100%|██████████| 100/100 [9:29:49<00:00, 341.90s/it]
+[2025-11-02 12:15:51,787] [    INFO] - Saving model checkpoint to tmp_train_mmarco.jsonl/bge-en-icl-paddle
+[2025-11-02 12:15:51,792] [    INFO] - tokenizer config file saved in tmp_train_mmarco.jsonl/bge-en-icl-paddle/tokenizer_config.json
+[2025-11-02 12:15:51,805] [    INFO] - Special tokens file saved in tmp_train_mmarco.jsonl/bge-en-icl-paddle/special_tokens_map.json
+[2025-11-02 12:15:51,830] [    INFO] - added tokens file saved in tmp_train_mmarco.jsonl/bge-en-icl-paddle/added_tokens.json
+[2025-11-02 12:15:52,475] [    INFO] - Configuration saved in tmp_train_mmarco.jsonl/bge-en-icl-paddle/config.json
+[2025-11-02 12:15:52,475] [    INFO] - ***** train metrics *****
+[2025-11-02 12:15:52,475] [    INFO] -   progress_or_epoch        =      0.064
+[2025-11-02 12:15:52,476] [    INFO] -   train_loss               =     0.2093
+[2025-11-02 12:15:52,476] [    INFO] -   train_runtime            = 9:29:49.75
+[2025-11-02 12:15:52,476] [    INFO] -   train_samples_per_second =     0.3744
+[2025-11-02 12:15:52,476] [    INFO] -   train_steps_per_second   =     0.0029
+sys:1: DeprecationWarning: builtin type swigvarlink has no __module__ attribute
+LAUNCH INFO 2025-11-02 12:15:55,487 Pod completed
+LAUNCH INFO 2025-11-02 12:15:55,487 Exit code 0

bge-en-icl-paddle/train_results.json

@@ -0,0 +1,7 @@
+{
+    "progress_or_epoch": 0.064,
+    "train_loss": 0.20930041134357452,
+    "train_runtime": 34189.7588,
+    "train_samples_per_second": 0.3744,
+    "train_steps_per_second": 0.0029
+}

bge-en-icl-paddle/trainer_state.json

@@ -0,0 +1,15 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "consumed_samples": 12800,
+  "epoch": 0.06399872002559949,
+  "global_step": 100,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [],
+  "max_steps": 100,
+  "num_train_epochs": 1,
+  "total_flos": 0,
+  "trial_name": null,
+  "trial_params": null
+}

bge-en-icl-paddle/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d438bcd4f02f0d15124a5678e5ad489a8329197fe3d3cd95c598a68718a64e8f
+size 3780