Hugging Face's logo

lzq2021
/
repllama_duretrieval_train

PaddlePaddle
Model card Files
xet
Community
lzq2021 commited on
Commit
fb2cfc4
·
verified ·
1 Parent(s): cba2e3c

Upload 12 files

Browse files
Files changed (13) hide show
  1. .gitattributes +1 -0
  2. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/all_results.json +7 -0
  3. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json +41 -0
  4. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/lora_config.json +32 -0
  5. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/lora_model_state.pdparams +3 -0
  6. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/runs/Oct29_01-19-08_b63e119a1648/vdlrecords.1761700806.log +0 -0
  7. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/sentencepiece.bpe.model +3 -0
  8. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/special_tokens_map.json +1 -0
  9. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/tokenizer_config.json +1 -0
  10. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/train_repllama-v1-7b-lora-passage-merge-then-to-paddle-float16_dureader_dual.train.jsonl.log +745 -0
  11. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/train_results.json +7 -0
  12. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/trainer_state.json +15 -0
  13. repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/training_args.bin +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/lora_model_state.pdparams filter=lfs diff=lfs merge=lfs -text
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/all_results.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "progress_or_epoch": 0.1482,
3
+ "train_loss": 0.7184327244758606,
4
+ "train_runtime": 40834.4907,
5
+ "train_samples_per_second": 0.3135,
6
+ "train_steps_per_second": 0.0024
7
+ }
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/ceph_home/lizhuoqun2021/hf_models/llama-2-7b",
3
+ "alibi": false,
4
+ "architectures": [
5
+ "LlamaForCausalLM"
6
+ ],
7
+ "attention_bias": false,
8
+ "attention_dropout": 0.0,
9
+ "bos_token_id": 1,
10
+ "dpo_config": null,
11
+ "dtype": "float16",
12
+ "eos_token_id": 2,
13
+ "hidden_act": "silu",
14
+ "hidden_size": 4096,
15
+ "immediate_clear_past_key_value": false,
16
+ "initializer_range": 0.02,
17
+ "intermediate_size": 11008,
18
+ "long_sequence_init_args": {},
19
+ "long_sequence_strategy_name": null,
20
+ "long_sequence_strategy_type": null,
21
+ "max_position_embeddings": 4096,
22
+ "mlp_bias": false,
23
+ "model_type": "llama",
24
+ "num_attention_heads": 32,
25
+ "num_hidden_layers": 32,
26
+ "num_key_value_heads": 32,
27
+ "pad_token_id": 0,
28
+ "paddlenlp_version": "3.0.0b4.post20250825",
29
+ "pretraining_tp": 1,
30
+ "rms_norm_eps": 1e-05,
31
+ "rope_scaling": null,
32
+ "rope_scaling_factor": 1.0,
33
+ "rope_scaling_type": null,
34
+ "rope_theta": 10000.0,
35
+ "seq_length": 2048,
36
+ "tie_word_embeddings": false,
37
+ "use_flash_attention_for_generation": false,
38
+ "use_last_token_for_generation": false,
39
+ "use_long_sequence_strategies": false,
40
+ "vocab_size": 32000
41
+ }
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/lora_config.json ADDED
@@ -0,0 +1,32 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "base_model_name_or_path": null,
3
+ "do_qat": false,
4
+ "dtype": "float16",
5
+ "enable_lora_list": null,
6
+ "head_dim": null,
7
+ "lora_alpha": 64,
8
+ "lora_dropout": 0.1,
9
+ "lora_plus_scale": 1.0,
10
+ "lora_use_mixer": false,
11
+ "loraga": false,
12
+ "lorapro": false,
13
+ "merge_weights": false,
14
+ "pissa": false,
15
+ "r": 32,
16
+ "rslora": false,
17
+ "scaling": 2.0,
18
+ "target_modules": [
19
+ ".*q_proj$",
20
+ ".*k_proj$",
21
+ ".*v_proj$",
22
+ ".*o_proj$",
23
+ ".*down_proj$",
24
+ ".*up_proj$",
25
+ ".*gate_proj$"
26
+ ],
27
+ "tensor_parallel_degree": 1,
28
+ "trainable_bias": null,
29
+ "trainable_modules": null,
30
+ "use_mora": false,
31
+ "use_quick_lora": false
32
+ }
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/lora_model_state.pdparams ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd3e1ea67535955b7c84a4439cbff5d52c707f793b451cdb0e6ce1270bd91d63
3
+ size 479881234
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/runs/Oct29_01-19-08_b63e119a1648/vdlrecords.1761700806.log ADDED
Binary file (68 kB). View file
 
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/sentencepiece.bpe.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/special_tokens_map.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false}, "pad_token": "<unk>"}
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false, "__type": "AddedToken"}, "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false, "__type": "AddedToken"}, "add_bos_token": true, "add_eos_token": false, "sp_model_kwargs": {}, "decode_with_prefix_space": false, "clean_up_tokenization_spaces": false, "legacy": true, "model_max_length": 1000000000000000019884624838656, "pad_token": null, "name_or_path": "/mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16", "from_slow": true, "_from_auto": true, "added_tokens_decoder": {}, "tokenizer_class": "LlamaTokenizer"}
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/train_repllama-v1-7b-lora-passage-merge-then-to-paddle-float16_dureader_dual.train.jsonl.log ADDED
@@ -0,0 +1,745 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ /mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md
2
+ warnings.warn(warning_message)
3
+ LAUNCH INFO 2025-10-29 01:19:03,022 ----------- Configuration ----------------------
4
+ LAUNCH INFO 2025-10-29 01:19:03,022 auto_cluster_config: 0
5
+ LAUNCH INFO 2025-10-29 01:19:03,022 auto_parallel_config: None
6
+ LAUNCH INFO 2025-10-29 01:19:03,022 auto_tuner_json: None
7
+ LAUNCH INFO 2025-10-29 01:19:03,022 devices: 0,1,2,3
8
+ LAUNCH INFO 2025-10-29 01:19:03,022 elastic_level: -1
9
+ LAUNCH INFO 2025-10-29 01:19:03,022 elastic_timeout: 30
10
+ LAUNCH INFO 2025-10-29 01:19:03,022 enable_gpu_log: True
11
+ LAUNCH INFO 2025-10-29 01:19:03,022 gloo_port: 6767
12
+ LAUNCH INFO 2025-10-29 01:19:03,022 host: None
13
+ LAUNCH INFO 2025-10-29 01:19:03,022 ips: None
14
+ LAUNCH INFO 2025-10-29 01:19:03,023 job_id: default
15
+ LAUNCH INFO 2025-10-29 01:19:03,023 legacy: False
16
+ LAUNCH INFO 2025-10-29 01:19:03,023 log_dir: log
17
+ LAUNCH INFO 2025-10-29 01:19:03,023 log_level: INFO
18
+ LAUNCH INFO 2025-10-29 01:19:03,023 log_overwrite: False
19
+ LAUNCH INFO 2025-10-29 01:19:03,023 master: None
20
+ LAUNCH INFO 2025-10-29 01:19:03,023 max_restart: 3
21
+ LAUNCH INFO 2025-10-29 01:19:03,023 nnodes: 1
22
+ LAUNCH INFO 2025-10-29 01:19:03,023 nproc_per_node: None
23
+ LAUNCH INFO 2025-10-29 01:19:03,023 rank: -1
24
+ LAUNCH INFO 2025-10-29 01:19:03,023 run_mode: collective
25
+ LAUNCH INFO 2025-10-29 01:19:03,023 server_num: None
26
+ LAUNCH INFO 2025-10-29 01:19:03,023 servers:
27
+ LAUNCH INFO 2025-10-29 01:19:03,023 sort_ip: False
28
+ LAUNCH INFO 2025-10-29 01:19:03,023 start_port: 6070
29
+ LAUNCH INFO 2025-10-29 01:19:03,023 trainer_num: None
30
+ LAUNCH INFO 2025-10-29 01:19:03,023 trainers:
31
+ LAUNCH INFO 2025-10-29 01:19:03,023 training_script: train.py
32
+ LAUNCH INFO 2025-10-29 01:19:03,023 training_script_args: ['--do_train', '--query_instruction_for_retrieval', 'query: ', '--passage_instruction_for_retrieval', '', '--model_name_or_path', '/mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16', '--output_dir', 'tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16', '--save_steps', '999999999', '--train_data', './data/dureader_dual.train.jsonl', '--fp16_opt_level', 'O2', '--fp16', '--per_device_train_batch_size', '1', '--gradient_accumulation_steps', '32', '--recompute', '--train_group_size', '4', '--learning_rate', '1e-4', '--query_max_len', '128', '--passage_max_len', '4096', '--num_train_epochs', '1', '--logging_steps', '1', '--overwrite_output_dir', '--negatives_cross_device', '--warmup_steps', '10', '--max_steps', '100', '--do_train', '--fine_tune_type', 'lora', '--sentence_pooling_method', 'last', '--sharding', 'stage3 offload', '--use_flash_attention', '--temperature', '0.01']
33
+ LAUNCH INFO 2025-10-29 01:19:03,023 with_gloo: 1
34
+ LAUNCH INFO 2025-10-29 01:19:03,023 --------------------------------------------------
35
+ LAUNCH INFO 2025-10-29 01:19:03,032 Job: default, mode collective, replicas 1[1:1], elastic False
36
+ LAUNCH INFO 2025-10-29 01:19:03,040 Run Pod: uymyap, replicas 4, status ready
37
+ LAUNCH INFO 2025-10-29 01:19:03,430 Watching Pod: uymyap, replicas 4, status running
38
+ /mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md
39
+ warnings.warn(warning_message)
40
+ /mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/_distutils_hack/__init__.py:30: UserWarning: Setuptools is replacing distutils. Support for replacing an already imported distutils is deprecated. In the future, this condition will fail. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
41
+ warnings.warn(
42
+ /mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddlenlp/trainer/training_args.py:1205: UserWarning: `offload` is not supported NOW!
43
+ warnings.warn("`offload` is not supported NOW!")
44
+ [2025-10-29 01:19:08,275] [ INFO] distributed_strategy.py:335 - distributed strategy initialized
45
+ ======================= Modified FLAGS detected =======================
46
+ FLAGS(name='FLAGS_cuda_cccl_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cuda_cccl/include/', default_value='')
47
+ FLAGS(name='FLAGS_cupti_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cuda_cupti/lib', default_value='')
48
+ FLAGS(name='FLAGS_cublas_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cublas/lib', default_value='')
49
+ FLAGS(name='FLAGS_nvidia_package_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia', default_value='')
50
+ FLAGS(name='FLAGS_curand_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/curand/lib', default_value='')
51
+ FLAGS(name='FLAGS_nccl_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/nccl/lib', default_value='')
52
+ FLAGS(name='FLAGS_selected_gpus', current_value='0', default_value='')
53
+ FLAGS(name='FLAGS_cusparse_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cusparse/lib', default_value='')
54
+ FLAGS(name='FLAGS_cusolver_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cusolver/lib', default_value='')
55
+ FLAGS(name='FLAGS_cudnn_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cudnn/lib', default_value='')
56
+ FLAGS(name='FLAGS_enable_pir_in_executor', current_value=True, default_value=False)
57
+ =======================================================================
58
+ I1029 01:19:08.276810 18932 tcp_store.cc:336] create libuv server at port: 43082
59
+ I1029 01:19:08.277513 18932 tcp_utils.cc:132] Successfully connected to 172.17.0.5:43082
60
+ I1029 01:19:08.953444 18932 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
61
+ I1029 01:19:08.953509 18932 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
62
+ [2025-10-29 01:19:08,954] [ INFO] topology.py:526 - Total 4 pipe comm group(s) create successfully!
63
+ W1029 01:19:08.956010 18932 gpu_resources.cc:114] Please NOTE: device: 0, GPU Compute Capability: 8.0, Driver API Version: 12.8, Runtime API Version: 12.8
64
+ /mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/distributed/communication/group.py:145: UserWarning: Current global rank 0 is not in group _default_pg10
65
+ warnings.warn(
66
+ [2025-10-29 01:19:08,959] [ INFO] topology.py:526 - Total 4 data comm group(s) create successfully!
67
+ [2025-10-29 01:19:08,959] [ INFO] topology.py:526 - Total 4 model comm group(s) create successfully!
68
+ I1029 01:19:08.959522 18932 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
69
+ I1029 01:19:08.959532 18932 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
70
+ [2025-10-29 01:19:08,959] [ INFO] topology.py:526 - Total 1 sharding comm group(s) create successfully!
71
+ I1029 01:19:08.959612 18932 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
72
+ I1029 01:19:08.959619 18932 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
73
+ [2025-10-29 01:19:08,959] [ INFO] topology.py:440 - HybridParallelInfo: rank_id: 0, mp_degree: 1, sharding_degree: 4, pp_degree: 1, dp_degree: 1, sep_degree: 1, mp_group: [0], sharding_group: [0, 1, 2, 3], pp_group: [0], dp_group: [0], sep:group: None, check/clip group: [0, 1, 2, 3]
74
+ [2025-10-29 01:19:08,959] [ INFO] - +==============================================================================+
75
+ | |
76
+ | DistributedStrategy Overview |
77
+ | |
78
+ +==============================================================================+
79
+ | a_sync=True <-> a_sync_configs |
80
+ +------------------------------------------------------------------------------+
81
+ | k_steps -1 |
82
+ | max_merge_var_num 1 |
83
+ | send_queue_size 16 |
84
+ | independent_recv_thread False |
85
+ | min_send_grad_num_before_recv 1 |
86
+ | thread_pool_size 1 |
87
+ | send_wait_times 1 |
88
+ | runtime_split_send_recv False |
89
+ | launch_barrier True |
90
+ | heter_worker_device_guard cpu |
91
+ | lr_decay_steps 10 |
92
+ | use_ps_gpu 0 |
93
+ | use_gpu_graph 0 |
94
+ +==============================================================================+
95
+ | Environment Flags, Communication Flags |
96
+ +------------------------------------------------------------------------------+
97
+ | mode 1 |
98
+ | elastic False |
99
+ | auto False |
100
+ | sync_nccl_allreduce True |
101
+ | nccl_comm_num 1 |
102
+ | use_hierarchical_allreduce False |
103
+ | hierarchical_allreduce_inter_nranks 1 |
104
+ | sync_batch_norm False |
105
+ | fuse_all_reduce_ops True |
106
+ | fuse_grad_size_in_MB 32 |
107
+ | fuse_grad_size_in_TFLOPS 50.0 |
108
+ | cudnn_exhaustive_search False |
109
+ | conv_workspace_size_limit 512 |
110
+ | cudnn_batchnorm_spatial_persistent False |
111
+ | fp16_allreduce False |
112
+ | last_comm_group_size_MB 1.0 |
113
+ | find_unused_parameters False |
114
+ | without_graph_optimization True |
115
+ | fuse_grad_size_in_num 8 |
116
+ | calc_comm_same_stream False |
117
+ | asp False |
118
+ | fuse_grad_merge False |
119
+ | semi_auto False |
120
+ | adam_d2sum False |
121
+ | auto_search False |
122
+ | heter_ccl_mode False |
123
+ | is_fl_ps_mode False |
124
+ | with_coordinator False |
125
+ | split_data True |
126
+ | downpour_table_param [] |
127
+ | fs_client_param |
128
+ +==============================================================================+
129
+ | Build Strategy |
130
+ +------------------------------------------------------------------------------+
131
+ | fuse_elewise_add_act_ops False |
132
+ | fuse_bn_act_ops False |
133
+ | fuse_relu_depthwise_conv False |
134
+ | fuse_broadcast_ops False |
135
+ | fuse_all_optimizer_ops False |
136
+ | enable_inplace False |
137
+ | enable_backward_optimizer_op_deps True |
138
+ | cache_runtime_context False |
139
+ | fuse_bn_add_act_ops True |
140
+ | enable_auto_fusion False |
141
+ | enable_addto False |
142
+ | allow_cuda_graph_capture False |
143
+ | reduce_strategy 0 |
144
+ | fuse_gemm_epilogue False |
145
+ | debug_graphviz_path |
146
+ | fused_attention False |
147
+ | fused_feedforward False |
148
+ | fuse_dot_product_attention False |
149
+ | fuse_resunit False |
150
+ +==============================================================================+
151
+ 
152
+ [2025-10-29 01:19:08,960] [ INFO] - The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).
153
+ [2025-10-29 01:19:08,960] [ WARNING] - Process rank: 0, device: gpu, distributed training: True, 16-bits training: True
154
+ [2025-10-29 01:19:08,961] [ INFO] - Training/evaluation parameters RetrieverTrainingArguments(
155
+ adam_beta1=0.9,
156
+ adam_beta2=0.999,
157
+ adam_epsilon=1e-08,
158
+ amp_custom_black_list=None,
159
+ amp_custom_white_list=None,
160
+ amp_master_grad=False,
161
+ auto_parallel_resume_form_hybrid_parallel=False,
162
+ bf16=False,
163
+ bf16_full_eval=False,
164
+ ckpt_quant_stage=O0,
165
+ context_parallel_degree=1,
166
+ count_trained_tokens=False,
167
+ data_parallel_config=,
168
+ dataloader_drop_last=False,
169
+ dataloader_num_workers=0,
170
+ dataloader_shuffle=True,
171
+ ddp_find_unused_parameters=None,
172
+ device=gpu,
173
+ disable_tqdm=False,
174
+ distributed_dataloader=False,
175
+ do_eval=False,
176
+ do_export=False,
177
+ do_predict=False,
178
+ do_train=True,
179
+ enable_auto_parallel=False,
180
+ enable_zero_cost_checkpoint=False,
181
+ eval_accumulation_steps=None,
182
+ eval_steps=None,
183
+ evaluation_strategy=IntervalStrategy.NO,
184
+ expert_max_capacity=4294967296,
185
+ expert_min_capacity=1,
186
+ expert_parallel_degree=1,
187
+ expert_tensor_parallel_degree=1,
188
+ fine_tune_type=lora,
189
+ fix_position_embedding=False,
190
+ flash_device_save_steps=0,
191
+ flatten_param_grads=False,
192
+ force_reshard_pp=False,
193
+ fp16=True,
194
+ fp16_full_eval=False,
195
+ fp16_opt_level=O2,
196
+ fuse_sequence_parallel_allreduce=False,
197
+ gradient_accumulation_steps=32,
198
+ greater_is_better=None,
199
+ hybrid_parallel_topo_order=pp_first,
200
+ ignore_data_skip=False,
201
+ ignore_load_lr_and_optim=False,
202
+ ignore_save_lr_and_optim=False,
203
+ label_names=None,
204
+ lazy_data_processing=True,
205
+ learning_rate=0.0001,
206
+ load_best_model_at_end=False,
207
+ load_sharded_model=False,
208
+ local_rank=0,
209
+ log_on_each_node=True,
210
+ logging_dir=tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/runs/Oct29_01-19-08_b63e119a1648,
211
+ logging_first_step=False,
212
+ logging_steps=1,
213
+ logging_strategy=IntervalStrategy.STEPS,
214
+ lr_end=1e-07,
215
+ lr_scheduler_type=SchedulerType.LINEAR,
216
+ margin=0.2,
217
+ matryoshka_dims=[64, 128, 256, 512, 768],
218
+ matryoshka_loss_weights=[1, 1, 1, 1, 1],
219
+ max_evaluate_steps=-1,
220
+ max_grad_norm=1.0,
221
+ max_steps=100,
222
+ metric_for_best_model=None,
223
+ metrics_output_path=None,
224
+ min_lr=0.0,
225
+ minimum_eval_times=None,
226
+ nccl_comm_group_config=None,
227
+ negatives_cross_device=True,
228
+ no_cuda=False,
229
+ num_cycles=0.5,
230
+ num_train_epochs=1.0,
231
+ offload_optim=False,
232
+ optim=OptimizerNames.ADAMW,
233
+ ordered_save_group_size=0,
234
+ output_dir=tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16,
235
+ output_signal_dir=tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16,
236
+ overwrite_output_dir=True,
237
+ pad_token_id=0,
238
+ past_index=-1,
239
+ pdc_download_ckpt=False,
240
+ pdc_download_timeout=300,
241
+ per_device_eval_batch_size=8,
242
+ per_device_train_batch_size=1,
243
+ pipeline_parallel_config=,
244
+ pipeline_parallel_degree=1,
245
+ power=1.0,
246
+ prediction_loss_only=False,
247
+ recompute=True,
248
+ refined_recompute={},
249
+ release_grads=False,
250
+ remove_unused_columns=True,
251
+ report_to=['visualdl'],
252
+ resume_from_checkpoint=None,
253
+ run_name=tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16,
254
+ save_on_each_node=False,
255
+ save_rng_states=True,
256
+ save_sharded_model=False,
257
+ save_sharding_stage1_model_include_freeze_params=False,
258
+ save_steps=999999999,
259
+ save_strategy=IntervalStrategy.STEPS,
260
+ save_tokenizer=True,
261
+ save_total_limit=None,
262
+ scale_loss=32768,
263
+ seed=42,
264
+ sentence_pooling_method=last,
265
+ sep_parallel_degree=1,
266
+ sequence_parallel=False,
267
+ sequence_parallel_config=,
268
+ sharding=[<ShardingOption.FULL_SHARD: 'stage3'>, <ShardingOption.OFFLOAD: 'offload'>],
269
+ sharding_comm_buffer_size_MB=-1,
270
+ sharding_degree=-1,
271
+ sharding_parallel_config=,
272
+ sharding_parallel_degree=4,
273
+ sharding_parallel_mesh_dimension=dp,
274
+ skip_data_intervals=None,
275
+ skip_memory_metrics=True,
276
+ skip_profile_timer=True,
277
+ split_inputs_sequence_dim=True,
278
+ split_norm_comm=False,
279
+ temperature=0.01,
280
+ tensor_parallel_config=,
281
+ tensor_parallel_degree=1,
282
+ tensorwise_offload_optimizer=False,
283
+ to_static=False,
284
+ unified_checkpoint=False,
285
+ unified_checkpoint_config=,
286
+ use_async_save=False,
287
+ use_expert_parallel=False,
288
+ use_inbatch_neg=False,
289
+ use_lowprecision_moment=False,
290
+ use_matryoshka=False,
291
+ wandb_api_key=None,
292
+ wandb_http_proxy=None,
293
+ warmup_ratio=0.0,
294
+ warmup_steps=10,
295
+ weight_decay=0.0,
296
+ zcc_ema_interval=1,
297
+ zcc_pipeline_hooks_capacity_usage=0.6,
298
+ zcc_save_ema_coef=None,
299
+ zcc_workers_num=3,
300
+ )
301
+ [2025-10-29 01:19:08,961] [ INFO] - Model parameters ModelArguments(model_name_or_path='/mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16', tokenizer_name=None, normalized=True, use_flash_attention=True)
302
+ [2025-10-29 01:19:08,961] [ INFO] - Data parameters DataArguments(train_data='./data/dureader_dual.train.jsonl', train_group_size=4, query_max_len=128, passage_max_len=4096, max_example_num_per_dataset=100000000, query_instruction_for_retrieval='query: ', passage_instruction_for_retrieval='')
303
+ [2025-10-29 01:19:08,962] [ INFO] - The global seed is set to 42, local seed is set to 46 and random seed is set to 42.
304
+ [2025-10-29 01:19:08,975] [ INFO] - We are using <class 'paddlenlp.transformers.llama.modeling.LlamaModel'> to load '/mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16'.
305
+ [2025-10-29 01:19:08,975] [ INFO] - Loading configuration file /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json
306
+ [2025-10-29 01:19:08,978] [ INFO] - Loading weights file /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/model_state.pdparams.index.json
307
+ file: /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/model_state.pdparams.index.json is paddle weight.
308
+
309
+ Loading checkpoint shards: 0%| | 0/3 [00:00<?, ?it/s]
310
+ Loading checkpoint shards: 33%|███▎ | 1/3 [00:08<00:16, 8.12s/it]
311
+ Loading checkpoint shards: 67%|██████▋ | 2/3 [00:16<00:08, 8.03s/it]
312
+ Loading checkpoint shards: 100%|██████████| 3/3 [00:21<00:00, 6.99s/it]
313
+ Loading checkpoint shards: 100%|██████████| 3/3 [00:21<00:00, 7.28s/it]
314
+ [2025-10-29 01:19:31,329] [ WARNING] - Some weights of the model checkpoint at /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16 were not used when initializing LlamaModel: ['lm_head.weight']
315
+ - This IS expected if you are initializing LlamaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
316
+ - This IS NOT expected if you are initializing LlamaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
317
+ [2025-10-29 01:19:31,346] [ INFO] - All the weights of LlamaModel were initialized from the model checkpoint at /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16.
318
+ If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaModel for predictions without further training.
319
+ [2025-10-29 01:19:31,349] [ INFO] - Loading configuration file /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json
320
+ [2025-10-29 01:19:31,738] [ WARNING] - Reset tensor_parallel_degree of lora_config to 1.
321
+ [2025-10-29 01:19:31,738] [ INFO] - Mark only lora and trainable_module as trainable.
322
+ [2025-10-29 01:19:31,768] [ DEBUG] - Frozen parameters: 1.98e+10 || Trainable parameters:2.40e+08 || Total parameters:2.01e+10|| Trainable:1.20%
323
+ [2025-10-29 01:19:31,815] [ INFO] - The global seed is set to 42, local seed is set to 46 and random seed is set to 42.
324
+ [2025-10-29 01:19:31,893] [ INFO] - max_steps is given, it will override any value given in num_train_epochs
325
+ [2025-10-29 01:19:31,893] [ INFO] - Using half precision
326
+ [2025-10-29 01:19:31,918] [ DEBUG] - ============================================================
327
+ [2025-10-29 01:19:31,918] [ DEBUG] - Training Configuration Arguments 
328
+ [2025-10-29 01:19:31,918] [ DEBUG] - paddle commit id : dc312219f1498234f178a3b6a93e9f1f4ddebcb8
329
+ [2025-10-29 01:19:31,918] [ DEBUG] - paddlenlp commit id : 0bb61071d051adf6207199d6493b08079d58f8b2.dirty
330
+ [2025-10-29 01:19:31,920] [ DEBUG] - _no_sync_in_gradient_accumulation: True
331
+ [2025-10-29 01:19:31,922] [ DEBUG] - adam_beta1 : 0.9
332
+ [2025-10-29 01:19:31,922] [ DEBUG] - adam_beta2 : 0.999
333
+ [2025-10-29 01:19:31,922] [ DEBUG] - adam_epsilon : 1e-08
334
+ [2025-10-29 01:19:31,923] [ DEBUG] - amp_custom_black_list : None
335
+ [2025-10-29 01:19:31,923] [ DEBUG] - amp_custom_white_list : None
336
+ [2025-10-29 01:19:31,923] [ DEBUG] - amp_master_grad : False
337
+ [2025-10-29 01:19:31,940] [ DEBUG] - auto_parallel_resume_form_hybrid_parallel: False
338
+ [2025-10-29 01:19:31,940] [ DEBUG] - bf16 : False
339
+ [2025-10-29 01:19:31,940] [ DEBUG] - bf16_full_eval : False
340
+ [2025-10-29 01:19:31,942] [ DEBUG] - ckpt_quant_stage : O0
341
+ [2025-10-29 01:19:31,942] [ DEBUG] - context_parallel_degree : 1
342
+ [2025-10-29 01:19:31,942] [ DEBUG] - count_trained_tokens : False
343
+ [2025-10-29 01:19:31,944] [ DEBUG] - current_device : gpu:0
344
+ [2025-10-29 01:19:31,944] [ DEBUG] - data_parallel_config : 
345
+ [2025-10-29 01:19:31,944] [ DEBUG] - data_parallel_degree : 1
346
+ [2025-10-29 01:19:31,946] [ DEBUG] - data_parallel_rank : 0
347
+ [2025-10-29 01:19:31,946] [ DEBUG] - dataloader_drop_last : False
348
+ [2025-10-29 01:19:31,946] [ DEBUG] - dataloader_num_workers : 0
349
+ [2025-10-29 01:19:31,947] [ DEBUG] - dataloader_shuffle : True
350
+ [2025-10-29 01:19:31,947] [ DEBUG] - dataset_rank : 0
351
+ [2025-10-29 01:19:31,947] [ DEBUG] - dataset_world_size : 4
352
+ [2025-10-29 01:19:31,948] [ DEBUG] - ddp_find_unused_parameters : None
353
+ [2025-10-29 01:19:31,948] [ DEBUG] - device : gpu
354
+ [2025-10-29 01:19:31,948] [ DEBUG] - disable_tqdm : False
355
+ [2025-10-29 01:19:31,948] [ DEBUG] - distributed_dataloader : False
356
+ [2025-10-29 01:19:31,948] [ DEBUG] - do_eval : False
357
+ [2025-10-29 01:19:31,949] [ DEBUG] - do_export : False
358
+ [2025-10-29 01:19:31,949] [ DEBUG] - do_predict : False
359
+ [2025-10-29 01:19:31,951] [ DEBUG] - do_train : True
360
+ [2025-10-29 01:19:31,951] [ DEBUG] - enable_auto_parallel : False
361
+ [2025-10-29 01:19:31,953] [ DEBUG] - enable_zero_cost_checkpoint : False
362
+ [2025-10-29 01:19:31,953] [ DEBUG] - eval_accumulation_steps : None
363
+ [2025-10-29 01:19:31,954] [ DEBUG] - eval_batch_size : 8
364
+ [2025-10-29 01:19:31,954] [ DEBUG] - eval_steps : None
365
+ [2025-10-29 01:19:31,956] [ DEBUG] - evaluation_strategy : IntervalStrategy.NO
366
+ [2025-10-29 01:19:31,956] [ DEBUG] - expert_max_capacity : 4294967296
367
+ [2025-10-29 01:19:31,958] [ DEBUG] - expert_min_capacity : 1
368
+ [2025-10-29 01:19:31,958] [ DEBUG] - expert_parallel_degree : 1
369
+ [2025-10-29 01:19:31,958] [ DEBUG] - expert_tensor_parallel_degree : 1
370
+ [2025-10-29 01:19:31,958] [ DEBUG] - fine_tune_type : lora
371
+ [2025-10-29 01:19:31,959] [ DEBUG] - fix_position_embedding : False
372
+ [2025-10-29 01:19:31,959] [ DEBUG] - flash_device_save_steps : 0
373
+ [2025-10-29 01:19:31,959] [ DEBUG] - flatten_param_grads : False
374
+ [2025-10-29 01:19:31,960] [ DEBUG] - force_reshard_pp : False
375
+ [2025-10-29 01:19:31,961] [ DEBUG] - fp16 : True
376
+ [2025-10-29 01:19:31,962] [ DEBUG] - fp16_full_eval : False
377
+ [2025-10-29 01:19:31,962] [ DEBUG] - fp16_opt_level : O2
378
+ [2025-10-29 01:19:31,963] [ DEBUG] - fuse_sequence_parallel_allreduce: False
379
+ [2025-10-29 01:19:31,963] [ DEBUG] - gradient_accumulation_steps : 32
380
+ [2025-10-29 01:19:31,964] [ DEBUG] - greater_is_better : None
381
+ [2025-10-29 01:19:31,965] [ DEBUG] - hybrid_parallel_topo_order : pp_first
382
+ [2025-10-29 01:19:31,965] [ DEBUG] - ignore_data_skip : False
383
+ [2025-10-29 01:19:31,965] [ DEBUG] - ignore_load_lr_and_optim : False
384
+ [2025-10-29 01:19:31,999] [ DEBUG] - ignore_save_lr_and_optim : False
385
+ [2025-10-29 01:19:31,999] [ DEBUG] - label_names : None
386
+ [2025-10-29 01:19:31,999] [ DEBUG] - lazy_data_processing : True
387
+ [2025-10-29 01:19:32,001] [ DEBUG] - learning_rate : 0.0001
388
+ [2025-10-29 01:19:32,001] [ DEBUG] - load_best_model_at_end : False
389
+ [2025-10-29 01:19:32,003] [ DEBUG] - load_sharded_model : False
390
+ [2025-10-29 01:19:32,004] [ DEBUG] - local_process_index : 0
391
+ [2025-10-29 01:19:32,006] [ DEBUG] - local_rank : 0
392
+ [2025-10-29 01:19:32,006] [ DEBUG] - log_level : -1
393
+ [2025-10-29 01:19:32,008] [ DEBUG] - log_level_replica : -1
394
+ [2025-10-29 01:19:32,008] [ DEBUG] - log_on_each_node : True
395
+ [2025-10-29 01:19:32,009] [ DEBUG] - logging_dir : tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/runs/Oct29_01-19-08_b63e119a1648
396
+ [2025-10-29 01:19:32,009] [ DEBUG] - logging_first_step : False
397
+ [2025-10-29 01:19:32,011] [ DEBUG] - logging_steps : 1
398
+ [2025-10-29 01:19:32,011] [ DEBUG] - logging_strategy : IntervalStrategy.STEPS
399
+ [2025-10-29 01:19:32,012] [ DEBUG] - logical_process_index : 0
400
+ [2025-10-29 01:19:32,041] [ DEBUG] - lr_end : 1e-07
401
+ [2025-10-29 01:19:32,041] [ DEBUG] - lr_scheduler_type : SchedulerType.LINEAR
402
+ [2025-10-29 01:19:32,073] [ DEBUG] - margin : 0.2
403
+ [2025-10-29 01:19:32,073] [ DEBUG] - matryoshka_dims : [64, 128, 256, 512, 768]
404
+ [2025-10-29 01:19:32,075] [ DEBUG] - matryoshka_loss_weights : [1, 1, 1, 1, 1]
405
+ [2025-10-29 01:19:32,075] [ DEBUG] - max_evaluate_steps : -1
406
+ [2025-10-29 01:19:32,076] [ DEBUG] - max_grad_norm : 1.0
407
+ [2025-10-29 01:19:32,077] [ DEBUG] - max_steps : 100
408
+ [2025-10-29 01:19:32,078] [ DEBUG] - metric_for_best_model : None
409
+ [2025-10-29 01:19:32,078] [ DEBUG] - metrics_output_path : None
410
+ [2025-10-29 01:19:32,080] [ DEBUG] - min_lr : 0.0
411
+ [2025-10-29 01:19:32,080] [ DEBUG] - minimum_eval_times : None
412
+ [2025-10-29 01:19:32,081] [ DEBUG] - nccl_comm_group_config : None
413
+ [2025-10-29 01:19:32,081] [ DEBUG] - negatives_cross_device : True
414
+ [2025-10-29 01:19:32,083] [ DEBUG] - no_cuda : False
415
+ [2025-10-29 01:19:32,083] [ DEBUG] - num_cycles : 0.5
416
+ [2025-10-29 01:19:32,084] [ DEBUG] - num_train_epochs : 1.0
417
+ [2025-10-29 01:19:32,084] [ DEBUG] - offload_optim : False
418
+ [2025-10-29 01:19:32,086] [ DEBUG] - optim : OptimizerNames.ADAMW
419
+ [2025-10-29 01:19:32,088] [ DEBUG] - optimizer_name_suffix : shard00
420
+ [2025-10-29 01:19:32,089] [ DEBUG] - ordered_save_group_size : 0
421
+ [2025-10-29 01:19:32,089] [ DEBUG] - output_dir : tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16
422
+ [2025-10-29 01:19:32,091] [ DEBUG] - output_signal_dir : tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16
423
+ [2025-10-29 01:19:32,091] [ DEBUG] - overwrite_output_dir : True
424
+ [2025-10-29 01:19:32,116] [ DEBUG] - pad_token_id : 0
425
+ [2025-10-29 01:19:32,116] [ DEBUG] - past_index : -1
426
+ [2025-10-29 01:19:32,118] [ DEBUG] - pdc_download_ckpt : False
427
+ [2025-10-29 01:19:32,118] [ DEBUG] - pdc_download_timeout : 300
428
+ [2025-10-29 01:19:32,120] [ DEBUG] - per_device_eval_batch_size : 8
429
+ [2025-10-29 01:19:32,120] [ DEBUG] - per_device_train_batch_size : 1
430
+ [2025-10-29 01:19:32,135] [ DEBUG] - pipeline_parallel_config : 
431
+ [2025-10-29 01:19:32,135] [ DEBUG] - pipeline_parallel_degree : 1
432
+ [2025-10-29 01:19:32,137] [ DEBUG] - pipeline_parallel_rank : 0
433
+ [2025-10-29 01:19:32,137] [ DEBUG] - power : 1.0
434
+ [2025-10-29 01:19:32,139] [ DEBUG] - prediction_loss_only : False
435
+ [2025-10-29 01:19:32,139] [ DEBUG] - process_index : 0
436
+ [2025-10-29 01:19:32,140] [ DEBUG] - recompute : True
437
+ [2025-10-29 01:19:32,140] [ DEBUG] - refined_recompute : {}
438
+ [2025-10-29 01:19:32,166] [ DEBUG] - release_grads : False
439
+ [2025-10-29 01:19:32,166] [ DEBUG] - remove_unused_columns : True
440
+ [2025-10-29 01:19:32,169] [ DEBUG] - report_to : ['visualdl']
441
+ [2025-10-29 01:19:32,169] [ DEBUG] - resume_from_checkpoint : None
442
+ [2025-10-29 01:19:32,198] [ DEBUG] - run_name : tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16
443
+ [2025-10-29 01:19:32,198] [ DEBUG] - save_on_each_node : False
444
+ [2025-10-29 01:19:32,200] [ DEBUG] - save_rng_states : True
445
+ [2025-10-29 01:19:32,200] [ DEBUG] - save_sharded_model : False
446
+ [2025-10-29 01:19:32,201] [ DEBUG] - save_sharding_stage1_model_include_freeze_params: False
447
+ [2025-10-29 01:19:32,201] [ DEBUG] - save_steps : 999999999
448
+ [2025-10-29 01:19:32,215] [ DEBUG] - save_strategy : IntervalStrategy.STEPS
449
+ [2025-10-29 01:19:32,216] [ DEBUG] - save_tokenizer : True
450
+ [2025-10-29 01:19:32,219] [ DEBUG] - save_total_limit : None
451
+ [2025-10-29 01:19:32,221] [ DEBUG] - scale_loss : 32768
452
+ [2025-10-29 01:19:32,222] [ DEBUG] - seed : 42
453
+ [2025-10-29 01:19:32,224] [ DEBUG] - sentence_pooling_method : last
454
+ [2025-10-29 01:19:32,224] [ DEBUG] - sep_parallel_degree : 1
455
+ [2025-10-29 01:19:32,276] [ DEBUG] - sequence_parallel : False
456
+ [2025-10-29 01:19:32,278] [ DEBUG] - sequence_parallel_config : 
457
+ [2025-10-29 01:19:32,280] [ DEBUG] - sharding : [<ShardingOption.FULL_SHARD: 'stage3'>, <ShardingOption.OFFLOAD: 'offload'>]
458
+ [2025-10-29 01:19:32,281] [ DEBUG] - sharding_comm_buffer_size_MB : -1
459
+ [2025-10-29 01:19:32,283] [ DEBUG] - sharding_degree : -1
460
+ [2025-10-29 01:19:32,284] [ DEBUG] - sharding_parallel_config : 
461
+ [2025-10-29 01:19:32,286] [ DEBUG] - sharding_parallel_degree : 4
462
+ [2025-10-29 01:19:32,288] [ DEBUG] - sharding_parallel_mesh_dimension: dp
463
+ [2025-10-29 01:19:32,289] [ DEBUG] - sharding_parallel_rank : 0
464
+ [2025-10-29 01:19:32,291] [ DEBUG] - should_load_dataset : True
465
+ [2025-10-29 01:19:32,308] [ DEBUG] - should_load_sharding_stage1_model: False
466
+ [2025-10-29 01:19:32,311] [ DEBUG] - should_log : True
467
+ [2025-10-29 01:19:32,312] [ DEBUG] - should_save : True
468
+ [2025-10-29 01:19:32,314] [ DEBUG] - should_save_model_state : True
469
+ [2025-10-29 01:19:32,339] [ DEBUG] - should_save_model_with_tensor_fusion: False
470
+ [2025-10-29 01:19:32,341] [ DEBUG] - should_save_sharding_stage1_model: False
471
+ [2025-10-29 01:19:32,342] [ DEBUG] - skip_data_intervals : None
472
+ [2025-10-29 01:19:32,344] [ DEBUG] - skip_memory_metrics : True
473
+ [2025-10-29 01:19:32,345] [ DEBUG] - skip_profile_timer : True
474
+ [2025-10-29 01:19:32,348] [ DEBUG] - split_inputs_sequence_dim : True
475
+ [2025-10-29 01:19:32,350] [ DEBUG] - split_norm_comm : False
476
+ [2025-10-29 01:19:32,351] [ DEBUG] - temperature : 0.01
477
+ [2025-10-29 01:19:32,353] [ DEBUG] - tensor_parallel_config : 
478
+ [2025-10-29 01:19:32,354] [ DEBUG] - tensor_parallel_degree : 1
479
+ [2025-10-29 01:19:32,356] [ DEBUG] - tensor_parallel_rank : 0
480
+ [2025-10-29 01:19:32,358] [ DEBUG] - tensorwise_offload_optimizer : False
481
+ [2025-10-29 01:19:32,359] [ DEBUG] - to_static : False
482
+ [2025-10-29 01:19:32,361] [ DEBUG] - train_batch_size : 1
483
+ [2025-10-29 01:19:32,363] [ DEBUG] - unified_checkpoint : False
484
+ [2025-10-29 01:19:32,365] [ DEBUG] - unified_checkpoint_config : 
485
+ [2025-10-29 01:19:32,376] [ DEBUG] - use_async_save : False
486
+ [2025-10-29 01:19:32,400] [ DEBUG] - use_expert_parallel : False
487
+ [2025-10-29 01:19:32,402] [ DEBUG] - use_hybrid_parallel : True
488
+ [2025-10-29 01:19:32,403] [ DEBUG] - use_inbatch_neg : False
489
+ [2025-10-29 01:19:32,405] [ DEBUG] - use_lowprecision_moment : False
490
+ [2025-10-29 01:19:32,407] [ DEBUG] - use_matryoshka : False
491
+ [2025-10-29 01:19:32,409] [ DEBUG] - wandb_api_key : None
492
+ [2025-10-29 01:19:32,410] [ DEBUG] - wandb_http_proxy : None
493
+ [2025-10-29 01:19:32,412] [ DEBUG] - warmup_ratio : 0.0
494
+ [2025-10-29 01:19:32,413] [ DEBUG] - warmup_steps : 10
495
+ [2025-10-29 01:19:32,416] [ DEBUG] - weight_decay : 0.0
496
+ [2025-10-29 01:19:32,417] [ DEBUG] - weight_name_suffix : 
497
+ [2025-10-29 01:19:32,419] [ DEBUG] - world_size : 4
498
+ [2025-10-29 01:19:32,420] [ DEBUG] - zcc_ema_interval : 1
499
+ [2025-10-29 01:19:32,422] [ DEBUG] - zcc_pipeline_hooks_capacity_usage: 0.6
500
+ [2025-10-29 01:19:32,423] [ DEBUG] - zcc_save_ema_coef : None
501
+ [2025-10-29 01:19:32,425] [ DEBUG] - zcc_workers_num : 3
502
+ [2025-10-29 01:19:32,426] [ DEBUG] - 
503
+ [2025-10-29 01:19:32,428] [ INFO] - Starting training from resume_from_checkpoint : None
504
+ W1029 01:19:32.645128 18932 nccl_comm_context.cc:70] ncclCommInitRankConfigMemOpt is not supported.
505
+ [2025-10-29 01:19:35,293] [ WARNING] group_sharded.py:148 - the input of scaler is None, please ensure the logic of your scaler outside is same as GroupShardedScaler.
506
+ [2025-10-29 01:19:35,298] [ WARNING] group_sharded_stage3.py:195 - While using ClipGradByGlobalNorm in GroupShardedStage3, the grad clip of original optimizer will be changed.
507
+ W1029 01:19:35.324023 18932 nccl_comm_context.cc:70] ncclCommInitRankConfigMemOpt is not supported.
508
+ [2025-10-29 01:20:06,959] [ INFO] - [timelog] checkpoint loading time: 0.00s (2025-10-29 01:20:06) 
509
+ [2025-10-29 01:20:06,959] [ INFO] - ***** Running training *****
510
+ [2025-10-29 01:20:06,959] [ INFO] - Num examples = 86,395
511
+ [2025-10-29 01:20:06,959] [ INFO] - Num Epochs = 1
512
+ [2025-10-29 01:20:06,959] [ INFO] - Instantaneous batch size per device = 1
513
+ [2025-10-29 01:20:06,959] [ INFO] - Total train batch size (w. parallel, distributed & accumulation) = 128
514
+ [2025-10-29 01:20:06,959] [ INFO] - Gradient Accumulation steps = 32
515
+ [2025-10-29 01:20:06,959] [ INFO] - Total optimization steps = 100
516
+ [2025-10-29 01:20:06,959] [ INFO] - Total num train samples = 12,800
517
+ [2025-10-29 01:20:06,964] [ DEBUG] - Number of trainable parameters = 79,953,920 (per device)
518
+
519
+ TrainProcess: 0%| | 0/100 [00:00<?, ?it/s]W1029 01:20:09.566076 18932 multiply_fwd_func.cc:82] got different data type, run type promotion automatically, this may cause data type been changed.
520
+ W1029 01:20:37.437152 18932 dygraph_functions.cc:94044] got different data type, run type promotion automatically, this may cause data type been changed.
521
+
522
+ TrainProcess: 1%| | 1/100 [05:59<9:52:49, 359.29s/it][2025-10-29 01:26:06,473] [ INFO] - loss: 0.94686508, learning_rate: 1e-05, global_step: 1, current_memory_allocated: 12.27591347694397, current_memory_reserved: 70.4673011302948, max_memory_allocated: 57.78868627548218, max_memory_reserved: 70.4673011302948, interval_runtime: 359.4045, interval_samples_per_second: 0.3561, interval_steps_per_second: 0.0028, progress_or_epoch: 0.0015
523
+
524
+ TrainProcess: 2%|▏ | 2/100 [12:46<10:33:09, 387.65s/it][2025-10-29 01:32:53,887] [ INFO] - loss: 1.01574707, learning_rate: 2e-05, global_step: 2, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.4034, interval_samples_per_second: 0.3142, interval_steps_per_second: 0.0025, progress_or_epoch: 0.003
525
+
526
+ TrainProcess: 3%|▎ | 3/100 [19:38<10:44:07, 398.42s/it][2025-10-29 01:39:45,122] [ INFO] - loss: 0.87535763, learning_rate: 3e-05, global_step: 3, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.2425, interval_samples_per_second: 0.3113, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0044
527
+
528
+ TrainProcess: 4%|▍ | 4/100 [26:27<10:44:22, 402.74s/it][2025-10-29 01:46:34,475] [ INFO] - loss: 1.0627327, learning_rate: 4e-05, global_step: 4, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.3795, interval_samples_per_second: 0.3127, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0059
529
+
530
+ TrainProcess: 5%|▌ | 5/100 [33:17<10:41:58, 405.46s/it][2025-10-29 01:53:24,794] [ INFO] - loss: 1.16447449, learning_rate: 5e-05, global_step: 5, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.2573, interval_samples_per_second: 0.312, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0074
531
+ LAUNCH WARNING 2025-10-29 01:56:57,310 save gpu log failed: Command '['nvidia-smi', '--format=csv,noheader,nounits', '--query-gpu=index,utilization.gpu,memory.total,memory.used,memory.free,timestamp', '--id=0,1,2,3']' timed out after 3 seconds
532
+ LAUNCH WARNING 2025-10-29 01:58:01,856 save gpu log failed: Command '['nvidia-smi', '--format=csv,noheader,nounits', '--query-gpu=index,utilization.gpu,memory.total,memory.used,memory.free,timestamp', '--id=0,1,2,3']' timed out after 3 seconds
533
+
534
+ TrainProcess: 6%|▌ | 6/100 [40:08<10:38:02, 407.26s/it][2025-10-29 02:00:15,520] [ INFO] - loss: 0.95492172, learning_rate: 6e-05, global_step: 6, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.7395, interval_samples_per_second: 0.3116, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0089
535
+
536
+ TrainProcess: 7%|▋ | 7/100 [47:00<10:33:48, 408.90s/it][2025-10-29 02:07:07,820] [ INFO] - loss: 0.81427097, learning_rate: 7e-05, global_step: 7, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 412.3005, interval_samples_per_second: 0.3105, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0104
537
+
538
+ TrainProcess: 8%|▊ | 8/100 [53:53<10:28:38, 409.99s/it][2025-10-29 02:14:00,051] [ INFO] - loss: 0.85042, learning_rate: 8e-05, global_step: 8, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 412.3005, interval_samples_per_second: 0.3105, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0119
539
+
540
+ TrainProcess: 9%|▉ | 9/100 [1:00:44<10:22:32, 410.47s/it][2025-10-29 02:20:51,643] [ INFO] - loss: 1.06962204, learning_rate: 9e-05, global_step: 9, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.5406, interval_samples_per_second: 0.311, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0133
541
+
542
+ TrainProcess: 10%|█ | 10/100 [1:07:35<10:16:07, 410.75s/it][2025-10-29 02:27:43,024] [ INFO] - loss: 0.80554199, learning_rate: 0.0001, global_step: 10, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.3839, interval_samples_per_second: 0.3111, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0148
543
+
544
+ TrainProcess: 11%|█ | 11/100 [1:14:26<10:09:09, 410.67s/it][2025-10-29 02:34:33,504] [ INFO] - loss: 0.97095108, learning_rate: 9.889e-05, global_step: 11, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.4767, interval_samples_per_second: 0.3118, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0163
545
+
546
+ TrainProcess: 12%|█▏ | 12/100 [1:21:18<10:02:50, 411.03s/it][2025-10-29 02:41:25,329] [ INFO] - loss: 0.77588272, learning_rate: 9.778e-05, global_step: 12, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.858, interval_samples_per_second: 0.3108, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0178
547
+
548
+ TrainProcess: 13%|█▎ | 13/100 [1:28:09<9:56:07, 411.12s/it] [2025-10-29 02:48:16,888] [ INFO] - loss: 0.88829422, learning_rate: 9.667e-05, global_step: 13, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.3249, interval_samples_per_second: 0.3112, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0193
549
+
550
+ TrainProcess: 14%|█▍ | 14/100 [1:35:00<9:48:59, 410.92s/it][2025-10-29 02:55:07,123] [ INFO] - loss: 0.87327576, learning_rate: 9.556e-05, global_step: 14, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.4584, interval_samples_per_second: 0.3118, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0207
551
+
552
+ TrainProcess: 15%|█▌ | 15/100 [1:41:50<9:41:54, 410.76s/it][2025-10-29 03:01:57,616] [ INFO] - loss: 0.72288322, learning_rate: 9.444e-05, global_step: 15, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.3774, interval_samples_per_second: 0.3119, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0222
553
+
554
+ TrainProcess: 16%|█▌ | 16/100 [1:48:40<9:34:35, 410.42s/it][2025-10-29 03:08:47,169] [ INFO] - loss: 0.82181644, learning_rate: 9.333e-05, global_step: 16, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.6387, interval_samples_per_second: 0.3125, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0237
555
+
556
+ TrainProcess: 17%|█▋ | 17/100 [1:55:28<9:26:51, 409.77s/it][2025-10-29 03:15:35,388] [ INFO] - loss: 0.84968853, learning_rate: 9.222e-05, global_step: 17, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.2657, interval_samples_per_second: 0.3135, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0252
557
+
558
+ TrainProcess: 18%|█▊ | 18/100 [2:02:16<9:19:23, 409.31s/it][2025-10-29 03:22:23,629] [ INFO] - loss: 0.77752113, learning_rate: 9.111e-05, global_step: 18, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.2416, interval_samples_per_second: 0.3135, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0267
559
+ LAUNCH WARNING 2025-10-29 03:25:51,063 save gpu log failed: Command '['nvidia-smi', '--format=csv,noheader,nounits', '--query-gpu=index,utilization.gpu,memory.total,memory.used,memory.free,timestamp', '--id=0,1,2,3']' timed out after 3 seconds
560
+
561
+ TrainProcess: 19%|█▉ | 19/100 [2:09:06<9:12:56, 409.59s/it][2025-10-29 03:29:13,883] [ INFO] - loss: 0.77223778, learning_rate: 9e-05, global_step: 19, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.237, interval_samples_per_second: 0.312, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0281
562
+
563
+ TrainProcess: 20%|██ | 20/100 [2:15:56<9:06:01, 409.52s/it][2025-10-29 03:36:03,281] [ INFO] - loss: 0.62772465, learning_rate: 8.889e-05, global_step: 20, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.3668, interval_samples_per_second: 0.3127, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0296
564
+
565
+ TrainProcess: 21%|██ | 21/100 [2:22:45<8:59:01, 409.39s/it][2025-10-29 03:42:52,322] [ INFO] - loss: 0.82329845, learning_rate: 8.778e-05, global_step: 21, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.0859, interval_samples_per_second: 0.3129, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0311
566
+
567
+ TrainProcess: 22%|██▏ | 22/100 [2:29:32<8:51:21, 408.73s/it][2025-10-29 03:49:39,532] [ INFO] - loss: 0.90839386, learning_rate: 8.667e-05, global_step: 22, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.1952, interval_samples_per_second: 0.3143, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0326
568
+
569
+ TrainProcess: 23%|██▎ | 23/100 [2:36:21<8:44:47, 408.92s/it][2025-10-29 03:56:28,852] [ INFO] - loss: 0.89253807, learning_rate: 8.556e-05, global_step: 23, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.3598, interval_samples_per_second: 0.3127, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0341
570
+
571
+ TrainProcess: 24%|██▍ | 24/100 [2:43:09<8:37:34, 408.61s/it][2025-10-29 04:03:16,797] [ INFO] - loss: 0.75592804, learning_rate: 8.444e-05, global_step: 24, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.9023, interval_samples_per_second: 0.3138, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0356
572
+
573
+ TrainProcess: 25%|██▌ | 25/100 [2:49:56<8:30:05, 408.07s/it][2025-10-29 04:10:03,535] [ INFO] - loss: 0.68338966, learning_rate: 8.333e-05, global_step: 25, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.7777, interval_samples_per_second: 0.3147, interval_steps_per_second: 0.0025, progress_or_epoch: 0.037
574
+
575
+ TrainProcess: 26%|██▌ | 26/100 [2:56:46<8:23:56, 408.60s/it][2025-10-29 04:16:53,434] [ INFO] - loss: 0.90509558, learning_rate: 8.222e-05, global_step: 26, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.896, interval_samples_per_second: 0.3123, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0385
576
+
577
+ TrainProcess: 27%|██▋ | 27/100 [3:03:36<8:17:40, 409.05s/it][2025-10-29 04:23:43,491] [ INFO] - loss: 0.67132568, learning_rate: 8.111e-05, global_step: 27, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.038, interval_samples_per_second: 0.3122, interval_steps_per_second: 0.0024, progress_or_epoch: 0.04
578
+
579
+ TrainProcess: 28%|██▊ | 28/100 [3:10:26<8:11:19, 409.44s/it][2025-10-29 04:30:33,857] [ INFO] - loss: 0.697299, learning_rate: 8e-05, global_step: 28, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.3719, interval_samples_per_second: 0.3119, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0415
580
+
581
+ TrainProcess: 29%|██▉ | 29/100 [3:17:15<8:04:17, 409.27s/it][2025-10-29 04:37:22,716] [ INFO] - loss: 0.74137878, learning_rate: 7.889e-05, global_step: 29, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.8653, interval_samples_per_second: 0.3131, interval_steps_per_second: 0.0024, progress_or_epoch: 0.043
582
+
583
+ TrainProcess: 30%|███ | 30/100 [3:24:02<7:56:43, 408.63s/it][2025-10-29 04:44:09,904] [ INFO] - loss: 0.65124702, learning_rate: 7.778e-05, global_step: 30, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.119, interval_samples_per_second: 0.3144, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0444
584
+
585
+ TrainProcess: 31%|███ | 31/100 [3:30:49<7:49:07, 407.94s/it][2025-10-29 04:50:56,225] [ INFO] - loss: 0.71731186, learning_rate: 7.667e-05, global_step: 31, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.3317, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0459
586
+
587
+ TrainProcess: 32%|███▏ | 32/100 [3:37:36<7:42:17, 407.90s/it][2025-10-29 04:57:44,090] [ INFO] - loss: 0.66580772, learning_rate: 7.556e-05, global_step: 32, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.8312, interval_samples_per_second: 0.3139, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0474
588
+
589
+ TrainProcess: 33%|███▎ | 33/100 [3:44:24<7:35:14, 407.68s/it][2025-10-29 05:04:31,234] [ INFO] - loss: 0.78065109, learning_rate: 7.444e-05, global_step: 33, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.1591, interval_samples_per_second: 0.3144, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0489
590
+
591
+ TrainProcess: 34%|███▍ | 34/100 [3:51:11<7:28:19, 407.57s/it][2025-10-29 05:11:18,496] [ INFO] - loss: 0.67545986, learning_rate: 7.333e-05, global_step: 34, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.3343, interval_samples_per_second: 0.3142, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0504
592
+
593
+ TrainProcess: 35%|███▌ | 35/100 [3:58:02<7:22:42, 408.66s/it][2025-10-29 05:18:09,656] [ INFO] - loss: 0.68822193, learning_rate: 7.222e-05, global_step: 35, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.1897, interval_samples_per_second: 0.3113, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0519
594
+
595
+ TrainProcess: 36%|███▌ | 36/100 [4:04:53<7:16:36, 409.32s/it][2025-10-29 05:25:00,550] [ INFO] - loss: 0.53668976, learning_rate: 7.111e-05, global_step: 36, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.8365, interval_samples_per_second: 0.3116, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0533
596
+
597
+ TrainProcess: 37%|███▋ | 37/100 [4:11:43<7:10:08, 409.67s/it][2025-10-29 05:31:51,059] [ INFO] - loss: 0.72838783, learning_rate: 7e-05, global_step: 37, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.4895, interval_samples_per_second: 0.3118, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0548
598
+
599
+ TrainProcess: 38%|███▊ | 38/100 [4:18:34<7:03:27, 409.81s/it][2025-10-29 05:38:41,181] [ INFO] - loss: 0.90949631, learning_rate: 6.889e-05, global_step: 38, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.15, interval_samples_per_second: 0.3121, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0563
600
+
601
+ TrainProcess: 39%|███▉ | 39/100 [4:25:23<6:56:39, 409.82s/it][2025-10-29 05:45:31,061] [ INFO] - loss: 0.58823967, learning_rate: 6.778e-05, global_step: 39, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.8334, interval_samples_per_second: 0.3123, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0578
602
+
603
+ TrainProcess: 40%|████ | 40/100 [4:32:13<6:49:42, 409.70s/it][2025-10-29 05:52:20,430] [ INFO] - loss: 0.57150555, learning_rate: 6.667e-05, global_step: 40, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.4208, interval_samples_per_second: 0.3126, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0593
604
+
605
+ TrainProcess: 41%|████ | 41/100 [4:38:59<6:41:54, 408.72s/it][2025-10-29 05:59:06,824] [ INFO] - loss: 0.84312391, learning_rate: 6.556e-05, global_step: 41, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.4154, interval_samples_per_second: 0.3149, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0607
606
+ LAUNCH WARNING 2025-10-29 06:03:08,914 save gpu log failed: Command '['nvidia-smi', '--format=csv,noheader,nounits', '--query-gpu=index,utilization.gpu,memory.total,memory.used,memory.free,timestamp', '--id=0,1,2,3']' timed out after 3 seconds
607
+
608
+ TrainProcess: 42%|████▏ | 42/100 [4:45:50<6:35:37, 409.27s/it][2025-10-29 06:05:57,401] [ INFO] - loss: 0.71920586, learning_rate: 6.444e-05, global_step: 42, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.568, interval_samples_per_second: 0.3118, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0622
609
+
610
+ TrainProcess: 43%|████▎ | 43/100 [4:52:40<6:28:59, 409.46s/it][2025-10-29 06:12:47,310] [ INFO] - loss: 0.78603363, learning_rate: 6.333e-05, global_step: 43, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.893, interval_samples_per_second: 0.3123, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0637
611
+
612
+ TrainProcess: 44%|████▍ | 44/100 [4:59:30<6:22:24, 409.73s/it][2025-10-29 06:19:37,681] [ INFO] - loss: 0.63661385, learning_rate: 6.222e-05, global_step: 44, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.4082, interval_samples_per_second: 0.3119, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0652
613
+
614
+ TrainProcess: 45%|████▌ | 45/100 [5:06:21<6:15:50, 410.00s/it][2025-10-29 06:26:28,314] [ INFO] - loss: 0.71114922, learning_rate: 6.111e-05, global_step: 45, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.5851, interval_samples_per_second: 0.3118, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0667
615
+
616
+ TrainProcess: 46%|████▌ | 46/100 [5:13:16<6:10:19, 411.48s/it][2025-10-29 06:33:23,225] [ INFO] - loss: 0.66089058, learning_rate: 6e-05, global_step: 46, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 414.9208, interval_samples_per_second: 0.3085, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0682
617
+
618
+ TrainProcess: 47%|████▋ | 47/100 [5:20:10<6:04:14, 412.34s/it][2025-10-29 06:40:17,636] [ INFO] - loss: 0.55014992, learning_rate: 5.889e-05, global_step: 47, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 414.3665, interval_samples_per_second: 0.3089, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0696
619
+
620
+ TrainProcess: 48%|████▊ | 48/100 [5:27:03<5:57:25, 412.42s/it][2025-10-29 06:47:10,204] [ INFO] - loss: 0.84242439, learning_rate: 5.778e-05, global_step: 48, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 412.6153, interval_samples_per_second: 0.3102, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0711
621
+
622
+ TrainProcess: 49%|████▉ | 49/100 [5:33:56<5:50:45, 412.65s/it][2025-10-29 06:54:03,390] [ INFO] - loss: 0.72346115, learning_rate: 5.667e-05, global_step: 49, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 413.169, interval_samples_per_second: 0.3098, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0726
623
+
624
+ TrainProcess: 50%|█████ | 50/100 [5:40:47<5:43:34, 412.30s/it][2025-10-29 07:00:54,828] [ INFO] - loss: 0.74690247, learning_rate: 5.556e-05, global_step: 50, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.4818, interval_samples_per_second: 0.3111, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0741
625
+
626
+ TrainProcess: 51%|█████ | 51/100 [5:47:37<5:35:57, 411.38s/it][2025-10-29 07:07:44,097] [ INFO] - loss: 0.77119827, learning_rate: 5.444e-05, global_step: 51, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.2319, interval_samples_per_second: 0.3128, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0756
627
+
628
+ TrainProcess: 52%|█████▏ | 52/100 [5:54:23<5:28:01, 410.02s/it][2025-10-29 07:14:30,952] [ INFO] - loss: 0.89096069, learning_rate: 5.333e-05, global_step: 52, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.8692, interval_samples_per_second: 0.3146, interval_steps_per_second: 0.0025, progress_or_epoch: 0.077
629
+
630
+ TrainProcess: 53%|█████▎ | 53/100 [6:01:11<5:20:32, 409.20s/it][2025-10-29 07:21:18,200] [ INFO] - loss: 0.67089844, learning_rate: 5.222e-05, global_step: 53, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.2792, interval_samples_per_second: 0.3143, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0785
631
+
632
+ TrainProcess: 54%|█████▍ | 54/100 [6:07:59<5:13:31, 408.94s/it][2025-10-29 07:28:06,989] [ INFO] - loss: 0.7338028, learning_rate: 5.111e-05, global_step: 54, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.311, interval_samples_per_second: 0.3135, interval_steps_per_second: 0.0024, progress_or_epoch: 0.08
633
+
634
+ TrainProcess: 55%|█████▌ | 55/100 [6:14:46<5:06:15, 408.35s/it][2025-10-29 07:34:53,527] [ INFO] - loss: 0.55512047, learning_rate: 5e-05, global_step: 55, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.9824, interval_samples_per_second: 0.3145, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0815
635
+
636
+ TrainProcess: 56%|█████▌ | 56/100 [6:21:35<4:59:40, 408.66s/it][2025-10-29 07:41:42,911] [ INFO] - loss: 0.64567184, learning_rate: 4.889e-05, global_step: 56, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.3866, interval_samples_per_second: 0.3127, interval_steps_per_second: 0.0024, progress_or_epoch: 0.083
637
+
638
+ TrainProcess: 57%|█████▋ | 57/100 [6:28:25<4:53:02, 408.90s/it][2025-10-29 07:48:32,331] [ INFO] - loss: 0.71821022, learning_rate: 4.778e-05, global_step: 57, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.4435, interval_samples_per_second: 0.3126, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0844
639
+
640
+ TrainProcess: 58%|█████▊ | 58/100 [6:35:12<4:45:55, 408.46s/it][2025-10-29 07:55:19,825] [ INFO] - loss: 0.61783791, learning_rate: 4.667e-05, global_step: 58, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.4402, interval_samples_per_second: 0.3142, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0859
641
+
642
+ TrainProcess: 59%|█████▉ | 59/100 [6:42:01<4:39:08, 408.50s/it][2025-10-29 08:02:08,368] [ INFO] - loss: 0.56420708, learning_rate: 4.556e-05, global_step: 59, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.6034, interval_samples_per_second: 0.3133, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0874
643
+
644
+ TrainProcess: 60%|██████ | 60/100 [6:48:50<4:32:29, 408.73s/it][2025-10-29 08:08:57,670] [ INFO] - loss: 0.63338661, learning_rate: 4.444e-05, global_step: 60, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.2918, interval_samples_per_second: 0.3127, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0889
645
+
646
+ TrainProcess: 61%|██████ | 61/100 [6:55:40<4:25:50, 408.98s/it][2025-10-29 08:15:47,263] [ INFO] - loss: 0.6884079, learning_rate: 4.333e-05, global_step: 61, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.5333, interval_samples_per_second: 0.3126, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0904
647
+
648
+ TrainProcess: 62%|██████▏ | 62/100 [7:02:29<4:19:04, 409.06s/it][2025-10-29 08:22:36,477] [ INFO] - loss: 0.59747648, learning_rate: 4.222e-05, global_step: 62, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.2657, interval_samples_per_second: 0.3128, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0919
649
+
650
+ TrainProcess: 63%|██████▎ | 63/100 [7:09:19<4:12:21, 409.23s/it][2025-10-29 08:29:26,141] [ INFO] - loss: 0.82813263, learning_rate: 4.111e-05, global_step: 63, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.6102, interval_samples_per_second: 0.3125, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0933
651
+
652
+ TrainProcess: 64%|██████▍ | 64/100 [7:16:07<4:05:19, 408.88s/it][2025-10-29 08:36:14,163] [ INFO] - loss: 0.55834961, learning_rate: 4e-05, global_step: 64, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.0507, interval_samples_per_second: 0.3137, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0948
653
+
654
+ TrainProcess: 65%|██████▌ | 65/100 [7:22:57<3:58:43, 409.23s/it][2025-10-29 08:43:04,259] [ INFO] - loss: 0.48566818, learning_rate: 3.889e-05, global_step: 65, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.0705, interval_samples_per_second: 0.3121, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0963
655
+
656
+ TrainProcess: 66%|██████▌ | 66/100 [7:29:45<3:51:45, 408.99s/it][2025-10-29 08:49:52,700] [ INFO] - loss: 0.58024406, learning_rate: 3.778e-05, global_step: 66, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.4352, interval_samples_per_second: 0.3134, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0978
657
+
658
+ TrainProcess: 67%|██████▋ | 67/100 [7:36:34<3:44:51, 408.84s/it][2025-10-29 08:56:41,151] [ INFO] - loss: 0.87768173, learning_rate: 3.667e-05, global_step: 67, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.4668, interval_samples_per_second: 0.3134, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0993
659
+ LAUNCH WARNING 2025-10-29 09:02:50,018 save gpu log failed: Command '['nvidia-smi', '--format=csv,noheader,nounits', '--query-gpu=index,utilization.gpu,memory.total,memory.used,memory.free,timestamp', '--id=0,1,2,3']' timed out after 3 seconds
660
+
661
+ TrainProcess: 68%|██████▊ | 68/100 [7:43:23<3:38:04, 408.89s/it][2025-10-29 09:03:30,189] [ INFO] - loss: 0.56551361, learning_rate: 3.556e-05, global_step: 68, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.0097, interval_samples_per_second: 0.313, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1007
662
+
663
+ TrainProcess: 69%|██████▉ | 69/100 [7:50:13<3:31:26, 409.25s/it][2025-10-29 09:10:20,264] [ INFO] - loss: 0.67060947, learning_rate: 3.444e-05, global_step: 69, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.1039, interval_samples_per_second: 0.3121, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1022
664
+
665
+ TrainProcess: 70%|███████ | 70/100 [7:57:03<3:24:46, 409.55s/it][2025-10-29 09:17:10,521] [ INFO] - loss: 0.56711197, learning_rate: 3.333e-05, global_step: 70, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.2514, interval_samples_per_second: 0.312, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1037
666
+
667
+ TrainProcess: 71%|███████ | 71/100 [8:03:51<3:17:48, 409.25s/it][2025-10-29 09:23:59,097] [ INFO] - loss: 0.76115704, learning_rate: 3.222e-05, global_step: 71, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.5363, interval_samples_per_second: 0.3133, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1052
668
+
669
+ TrainProcess: 72%|███████▏ | 72/100 [8:10:43<3:11:16, 409.89s/it][2025-10-29 09:30:50,441] [ INFO] - loss: 0.57299423, learning_rate: 3.111e-05, global_step: 72, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.3813, interval_samples_per_second: 0.3111, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1067
670
+
671
+ TrainProcess: 73%|███████▎ | 73/100 [8:17:34<3:04:37, 410.29s/it][2025-10-29 09:37:41,648] [ INFO] - loss: 0.70865393, learning_rate: 3e-05, global_step: 73, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 411.2308, interval_samples_per_second: 0.3113, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1082
672
+
673
+ TrainProcess: 74%|███████▍ | 74/100 [8:24:27<2:58:10, 411.16s/it][2025-10-29 09:44:34,851] [ INFO] - loss: 0.66517663, learning_rate: 2.889e-05, global_step: 74, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 413.1931, interval_samples_per_second: 0.3098, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1096
674
+
675
+ TrainProcess: 75%|███████▌ | 75/100 [8:31:18<2:51:17, 411.11s/it][2025-10-29 09:51:25,832] [ INFO] - loss: 0.78123283, learning_rate: 2.778e-05, global_step: 75, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.9897, interval_samples_per_second: 0.3114, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1111
676
+
677
+ TrainProcess: 76%|███████▌ | 76/100 [8:38:08<2:44:14, 410.58s/it][2025-10-29 09:58:15,199] [ INFO] - loss: 0.70936012, learning_rate: 2.667e-05, global_step: 76, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.3611, interval_samples_per_second: 0.3127, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1126
678
+
679
+ TrainProcess: 77%|███████▋ | 77/100 [8:44:53<2:36:48, 409.05s/it][2025-10-29 10:05:00,606] [ INFO] - loss: 0.45618534, learning_rate: 2.556e-05, global_step: 77, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.469, interval_samples_per_second: 0.3157, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1141
680
+
681
+ TrainProcess: 78%|███████▊ | 78/100 [8:51:41<2:29:53, 408.82s/it][2025-10-29 10:11:48,872] [ INFO] - loss: 0.65037632, learning_rate: 2.444e-05, global_step: 78, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.2667, interval_samples_per_second: 0.3135, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1156
682
+
683
+ TrainProcess: 79%|███████▉ | 79/100 [8:58:32<2:23:17, 409.43s/it][2025-10-29 10:18:39,723] [ INFO] - loss: 0.62935638, learning_rate: 2.333e-05, global_step: 79, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.8472, interval_samples_per_second: 0.3116, interval_steps_per_second: 0.0024, progress_or_epoch: 0.117
684
+
685
+ TrainProcess: 80%|████████ | 80/100 [9:05:22<2:16:32, 409.60s/it][2025-10-29 10:25:29,771] [ INFO] - loss: 0.60315037, learning_rate: 2.222e-05, global_step: 80, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.0381, interval_samples_per_second: 0.3122, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1185
686
+
687
+ TrainProcess: 81%|████████ | 81/100 [9:12:12<2:09:44, 409.69s/it][2025-10-29 10:32:19,644] [ INFO] - loss: 0.66599083, learning_rate: 2.111e-05, global_step: 81, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.8682, interval_samples_per_second: 0.3123, interval_steps_per_second: 0.0024, progress_or_epoch: 0.12
688
+
689
+ TrainProcess: 82%|████████▏ | 82/100 [9:19:01<2:02:51, 409.53s/it][2025-10-29 10:39:08,805] [ INFO] - loss: 0.63538933, learning_rate: 2e-05, global_step: 82, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.18, interval_samples_per_second: 0.3128, interval_steps_per_second: 0.0024, progress_or_epoch: 0.1215
690
+
691
+ TrainProcess: 83%|████████▎ | 83/100 [9:25:47<1:55:40, 408.25s/it][2025-10-29 10:45:54,075] [ INFO] - loss: 0.82035637, learning_rate: 1.889e-05, global_step: 83, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.2325, interval_samples_per_second: 0.3159, interval_steps_per_second: 0.0025, progress_or_epoch: 0.123
692
+
693
+ TrainProcess: 84%|████████▍ | 84/100 [9:32:31<1:48:32, 407.03s/it][2025-10-29 10:52:38,286] [ INFO] - loss: 0.72618818, learning_rate: 1.778e-05, global_step: 84, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.2029, interval_samples_per_second: 0.3167, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1245
694
+
695
+ TrainProcess: 85%|████████▌ | 85/100 [9:39:17<1:41:42, 406.84s/it][2025-10-29 10:59:24,715] [ INFO] - loss: 0.46369886, learning_rate: 1.667e-05, global_step: 85, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.3909, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1259
696
+
697
+ TrainProcess: 86%|████████▌ | 86/100 [9:46:02<1:34:47, 406.28s/it][2025-10-29 11:06:09,687] [ INFO] - loss: 0.73223495, learning_rate: 1.556e-05, global_step: 86, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.9705, interval_samples_per_second: 0.3161, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1274
698
+ LAUNCH WARNING 2025-10-29 11:06:21,348 save gpu log failed: Command '['nvidia-smi', '--format=csv,noheader,nounits', '--query-gpu=index,utilization.gpu,memory.total,memory.used,memory.free,timestamp', '--id=0,1,2,3']' timed out after 3 seconds
699
+
700
+ TrainProcess: 87%|████████▋ | 87/100 [9:52:47<1:27:57, 405.94s/it][2025-10-29 11:12:54,799] [ INFO] - loss: 0.80208683, learning_rate: 1.444e-05, global_step: 87, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.1353, interval_samples_per_second: 0.3159, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1289
701
+
702
+ TrainProcess: 88%|████████▊ | 88/100 [9:59:33<1:21:10, 405.87s/it][2025-10-29 11:19:40,524] [ INFO] - loss: 0.7246666, learning_rate: 1.333e-05, global_step: 88, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.7109, interval_samples_per_second: 0.3155, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1304
703
+
704
+ TrainProcess: 89%|████████▉ | 89/100 [10:06:20<1:14:29, 406.29s/it][2025-10-29 11:26:27,822] [ INFO] - loss: 0.78582764, learning_rate: 1.222e-05, global_step: 89, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.2726, interval_samples_per_second: 0.3143, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1319
705
+
706
+ TrainProcess: 90%|█████████ | 90/100 [10:13:08<1:07:47, 406.77s/it][2025-10-29 11:33:15,694] [ INFO] - loss: 0.55720043, learning_rate: 1.111e-05, global_step: 90, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.8823, interval_samples_per_second: 0.3138, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1333
707
+
708
+ TrainProcess: 91%|█████████ | 91/100 [10:19:53<1:00:56, 406.33s/it][2025-10-29 11:40:00,989] [ INFO] - loss: 0.64777184, learning_rate: 1e-05, global_step: 91, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.3148, interval_samples_per_second: 0.3158, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1348
709
+
710
+ TrainProcess: 92%|█████████▏| 92/100 [10:26:38<54:05, 405.71s/it] [2025-10-29 11:46:45,247] [ INFO] - loss: 0.67457199, learning_rate: 8.889e-06, global_step: 92, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.2459, interval_samples_per_second: 0.3166, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1363
711
+
712
+ TrainProcess: 93%|█████████▎| 93/100 [10:33:22<47:17, 405.40s/it][2025-10-29 11:53:29,936] [ INFO] - loss: 0.5895319, learning_rate: 7.778e-06, global_step: 93, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.69, interval_samples_per_second: 0.3163, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1378
713
+
714
+ TrainProcess: 94%|█████████▍| 94/100 [10:40:07<40:30, 405.06s/it][2025-10-29 12:00:14,187] [ INFO] - loss: 0.69682932, learning_rate: 6.667e-06, global_step: 94, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.2835, interval_samples_per_second: 0.3166, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1393
715
+
716
+ TrainProcess: 95%|█████████▌| 95/100 [10:46:51<33:44, 404.96s/it][2025-10-29 12:06:58,942] [ INFO] - loss: 0.57092571, learning_rate: 5.556e-06, global_step: 95, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.7192, interval_samples_per_second: 0.3163, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1407
717
+
718
+ TrainProcess: 96%|█████████▌| 96/100 [10:53:35<26:58, 404.67s/it][2025-10-29 12:13:42,938] [ INFO] - loss: 0.34253502, learning_rate: 4.444e-06, global_step: 96, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 403.9836, interval_samples_per_second: 0.3168, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1422
719
+
720
+ TrainProcess: 97%|█████████▋| 97/100 [11:00:21<20:14, 404.99s/it][2025-10-29 12:20:28,713] [ INFO] - loss: 0.44466591, learning_rate: 3.333e-06, global_step: 97, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.7255, interval_samples_per_second: 0.3155, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1437
721
+
722
+ TrainProcess: 98%|█████████▊| 98/100 [11:07:06<13:29, 404.84s/it][2025-10-29 12:27:13,131] [ INFO] - loss: 0.57364511, learning_rate: 2.222e-06, global_step: 98, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.523, interval_samples_per_second: 0.3164, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1452
723
+
724
+ TrainProcess: 99%|█████████▉| 99/100 [11:13:50<06:44, 404.79s/it][2025-10-29 12:33:57,793] [ INFO] - loss: 0.63700104, learning_rate: 1.111e-06, global_step: 99, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.6643, interval_samples_per_second: 0.3163, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1467
725
+
726
+ TrainProcess: 100%|██████████| 100/100 [11:20:34<00:00, 404.47s/it][2025-10-29 12:40:41,469] [ INFO] - loss: 0.54217148, learning_rate: 0.0, global_step: 100, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 403.7166, interval_samples_per_second: 0.3171, interval_steps_per_second: 0.0025, progress_or_epoch: 0.1482
727
+ [2025-10-29 12:40:41,469] [ INFO] -
728
+ Training completed.
729
+ 
730
+ [2025-10-29 12:40:41,510] [ INFO] - train_runtime: 40834.4907, train_samples_per_second: 0.3135, train_steps_per_second: 0.0024, train_loss: 0.7184327244758606, progress_or_epoch: 0.1482
731
+
732
+ TrainProcess: 100%|██████████| 100/100 [11:20:34<00:00, 408.35s/it]
733
+ [2025-10-29 12:40:41,514] [ INFO] - Saving model checkpoint to tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16
734
+ [2025-10-29 12:40:41,545] [ INFO] - tokenizer config file saved in tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/tokenizer_config.json
735
+ [2025-10-29 12:40:41,549] [ INFO] - Special tokens file saved in tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/special_tokens_map.json
736
+ [2025-10-29 12:40:43,729] [ INFO] - Configuration saved in tmp_train/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json
737
+ [2025-10-29 12:40:43,729] [ INFO] - ***** train metrics *****
738
+ [2025-10-29 12:40:43,729] [ INFO] - progress_or_epoch = 0.1482
739
+ [2025-10-29 12:40:43,729] [ INFO] - train_loss = 0.7184
740
+ [2025-10-29 12:40:43,729] [ INFO] - train_runtime = 11:20:34.49
741
+ [2025-10-29 12:40:43,729] [ INFO] - train_samples_per_second = 0.3135
742
+ [2025-10-29 12:40:43,730] [ INFO] - train_steps_per_second = 0.0024
743
+ sys:1: DeprecationWarning: builtin type swigvarlink has no __module__ attribute
744
+ LAUNCH INFO 2025-10-29 12:40:53,477 Pod completed
745
+ LAUNCH INFO 2025-10-29 12:40:53,477 Exit code 0
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/train_results.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "progress_or_epoch": 0.1482,
3
+ "train_loss": 0.7184327244758606,
4
+ "train_runtime": 40834.4907,
5
+ "train_samples_per_second": 0.3135,
6
+ "train_steps_per_second": 0.0024
7
+ }
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/trainer_state.json ADDED
@@ -0,0 +1,15 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "consumed_samples": 12800,
5
+ "epoch": 0.14815500717625815,
6
+ "global_step": 100,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [],
10
+ "max_steps": 100,
11
+ "num_train_epochs": 1,
12
+ "total_flos": 0,
13
+ "trial_name": null,
14
+ "trial_params": null
15
+ }
repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2e15cb06616aeac59719c189e6f1fbdfa0e18ede618f4cc60b8a730301a0c042
3
+ size 3832