From f30449f30a61857b31875f493873682a765ff856 Mon Sep 17 00:00:00 2001
From: yanzhixiao <yanzhixiao@h-partners.com>
Date: Mon, 11 Aug 2025 17:33:06 +0800
Subject: [PATCH] add pipeline-v2

---
 tests/pipeline/pipe_run_v2.sh | 87 +++++++++++++++++++++++++++++++++++
 1 file changed, 87 insertions(+)
 create mode 100644 tests/pipeline/pipe_run_v2.sh

diff --git a/tests/pipeline/pipe_run_v2.sh b/tests/pipeline/pipe_run_v2.sh
new file mode 100644
index 000000000..c7785f719
--- /dev/null
+++ b/tests/pipeline/pipe_run_v2.sh
@@ -0,0 +1,87 @@
+# Setting
+# source ascend-toolkit 环境变量
+source /usr/local/Ascend/ascend-toolkit/set_env.sh
+
+# source atb库 环境变量
+source /usr/local/Ascend/nnal/atb/set_env.sh
+
+# 安装加速库
+git clone -b master https://gitee.com/ascend/MindSpeed.git
+cd MindSpeed
+
+# checkout commit from MindSpeed master in 2025.08.01
+git checkout 60f104c7
+pip install -r requirements.txt
+pip3 install -e .
+cd ..
+
+git clone -b master https://gitee.com/ascend/MindSpeed-LLM.git
+cd MindSpeed-LLM
+
+pip install -r requirements.txt
+
+# megatron core_v0.12.1
+cp -rf /home/master_branch/Megatron-LM/megatron ./
+
+# switch to the v2 framework
+find . -type f -name "*.py" -exec sed -i 's/from mindspeed_llm import megatron_adaptor/from mindspeed_llm import megatron_adaptor_v2/g' {} +
+sed -i 's/from mindspeed_llm.tasks import megatron_adaptor/from mindspeed_llm.tasks import megatron_adaptor_v2/g' mindspeed_llm/__init__.py
+sed -i 's/parser = process_args(parser)/parser = process_args_v2(parser)/g' mindspeed_llm/training/arguments.py
+
+# Main
+# step 1: define dir
+BASE_DIR=$(dirname "$(readlink -f "$0")")
+CURRENT_TIME=$(date "+%Y-%m-%d")
+BASELINE_DIR="$BASE_DIR/baseline"
+GENERATE_LOG_BASE_DIR="/$(echo "$BASE_DIR" | cut -d'/' -f2)/pipeline_log_v2"
+GENERATE_LOG_DIR="$GENERATE_LOG_BASE_DIR/$CURRENT_TIME"
+
+#mkdir cache to store product and will be removed after test
+mkdir -p "$GENERATE_LOG_DIR"
+touch "$GENERATE_LOG_DIR/exec_error.log"
+chmod a+w "$GENERATE_LOG_DIR/exec_error.log"
+echo "core0.12.0 Execution Results" > $GENERATE_LOG_DIR/exec_error.log
+
+# step 2: running scripts and execute `test_ci_pipeline.py` && running pytest
+find "$BASE_DIR" -mindepth 1 -maxdepth 1 -type d | while read -r dir; do
+    if [ -d "$dir" ]; then
+        find "$dir" -type f -name "*.sh" | while read -r file; do
+            filename=$(basename "$file")
+            extension="${filename##*.}"
+            name="${filename%.$extension}"
+            echo "running $file"
+            bash $file 2>&1 | tee "$GENERATE_LOG_DIR/$name.log"
+            SCRIPT_EXITCODE=${PIPESTATUS[0]}
+            if [ $SCRIPT_EXITCODE -eq 0 ]; then
+                # begin to execute the logic of compare
+                echo "$(dirname "$BASE_DIR")/test_tools/test_ci_st.py"
+                pytest -x $(dirname "$BASE_DIR")/test_tools/test_ci_st.py \
+                    --baseline-json $BASELINE_DIR/$name.json \
+                    --generate-log $GENERATE_LOG_DIR/$name.log \
+                    --generate-json $GENERATE_LOG_DIR/$name.json
+                PYTEST_EXITCODE=$?
+                if [ $PYTEST_EXITCODE -ne 0 ]; then
+                    echo "${name}.sh compare to baseline has failed, check it!" >> $GENERATE_LOG_DIR/exec_error.log
+                fi
+            else
+                echo "${name}.sh Script has failed. Exit!" >> $GENERATE_LOG_DIR/exec_error.log
+            fi
+        done
+
+        # python test testing
+        find "$dir" -type f -name "*.py" | while read -r file; do
+            echo "running $file"
+            tmp_file_name="${file#*MindSpeed-LLM/}"
+            file_name="${tmp_file_name//\//_}"
+            pytest -x --log-level=INFO "$file" 2>&1 | tee "${GENERATE_LOG_DIR}/${file_name}.log"
+            PYTEST_EXITCODE=${PIPESTATUS[0]}
+            if [ $PYTEST_EXITCODE -ne 0 ]; then
+                echo "$file has failed, check it!" >> "$GENERATE_LOG_DIR/exec_error.log"
+            fi
+        done
+    fi
+done
+
+rm -rf /data/cache/*
+echo "=================tar error log=================="
+tar -czvf "${GENERATE_LOG_BASE_DIR}/${CURRENT_TIME}.tar.gz" "${GENERATE_LOG_DIR}/"
\ No newline at end of file
-- 
Gitee