From d3af6f851a5a6380ba9c97d99801632eb19f89ce Mon Sep 17 00:00:00 2001 From: yanzhixiao Date: Sun, 20 Apr 2025 22:13:39 +0800 Subject: [PATCH] =?UTF-8?q?fix:=E5=88=A0=E9=99=A4pipeline=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E7=94=9F=E6=88=90=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/pipeline/common/test_process_instruction_data_lf.json | 2 +- .../pipeline/configs/ray_online_dpo_full_llama32_1b_tp1pp1.yaml | 1 - tests/pipeline/pipe_run.sh | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pipeline/common/test_process_instruction_data_lf.json b/tests/pipeline/common/test_process_instruction_data_lf.json index bfb24c8c4..072babf45 100644 --- a/tests/pipeline/common/test_process_instruction_data_lf.json +++ b/tests/pipeline/common/test_process_instruction_data_lf.json @@ -5,7 +5,7 @@ "input": "/data/tune_dataset/train-00000-of-00001-a09b74b3ef9c3b56.parquet", "tokenizer-type": "PretrainedFromHF", "handler-name": "AlpacaStyleInstructionHandler", - "output-prefix": "/data/tune_dataset/alpaca/alpaca", + "output-prefix": "/data/pipe_cache/alpaca", "overwrite-cache": null, "tokenizer-name-or-path": "/data/qwen-7b/", "workers": 4, diff --git a/tests/pipeline/configs/ray_online_dpo_full_llama32_1b_tp1pp1.yaml b/tests/pipeline/configs/ray_online_dpo_full_llama32_1b_tp1pp1.yaml index 23c5bf25f..ffddf2128 100644 --- a/tests/pipeline/configs/ray_online_dpo_full_llama32_1b_tp1pp1.yaml +++ b/tests/pipeline/configs/ray_online_dpo_full_llama32_1b_tp1pp1.yaml @@ -45,7 +45,6 @@ actor_rollout_ref: initial_loss_scale: 4096 finetune: true load: /data/ppo/llama-3.2-1b-instruct-tp1-pp1 - save: ./ckpt num_gpus_for_train: 1 num_gpus_for_infer: 1 pad_to_multiple_of: 1 diff --git a/tests/pipeline/pipe_run.sh b/tests/pipeline/pipe_run.sh index 881800c0a..f2ddd9aeb 100644 --- a/tests/pipeline/pipe_run.sh +++ b/tests/pipeline/pipe_run.sh @@ -70,3 +70,4 @@ find "$BASE_DIR" -mindepth 1 -maxdepth 1 -type d | while read -r dir; do done fi done +rm -rf /data/pipe_cache/* \ No newline at end of file -- Gitee