diff --git a/.gitmodules b/.gitmodules index b08433f072bf89f62edf88b3aff40d24c1040ea8..0c8727a91869b9afe6f5c50ff759ecb5fb45988c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "dynolog_npu/third_party/dynolog"] - path = dynolog_npu/third_party/dynolog +[submodule "msmonitor/third_party/dynolog"] + path = msmonitor/third_party/dynolog url = https://github.com/facebookincubator/dynolog.git diff --git a/OWNERS b/OWNERS index 775490f4b708f61c57619326b579a52fa49a9b86..1b8f63546de38bc966852e4e1b318ad68a1161af 100644 --- a/OWNERS +++ b/OWNERS @@ -10,6 +10,7 @@ approvers: - ly-qianxiao - blian - kun_8 +- uniteone reviewers: - lv-kaimeng - wo-wenjie @@ -22,4 +23,5 @@ reviewers: - kali20gakki - wjchuee - chenhao_1209 -- feng123www \ No newline at end of file +- feng123www +- uniteone \ No newline at end of file diff --git a/README.md b/README.md index 5ae0bf742fced7ed86452d03d013670cc3528316..96394cb1e2457054e8e2018760eb1c701f015d0b 100644 --- a/README.md +++ b/README.md @@ -12,19 +12,13 @@ ![Commit Activity](https://img.shields.io/badge/commit%20activity-high-red) ![License: Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue) -## [分析迁移工具](https://gitee.com/ascend/mstt/wikis/工具介绍/分析迁移工具/分析迁移工具介绍) +## [模型训练开发全流程](https://www.hiascend.com/software/mindstudio/training) -1. [脚本分析工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E5%88%86%E6%9E%90%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) +mstt包括精度工具（msprobe）和性能工具（msprof-analyze），分析迁移工具请参见[昇腾社区](https://www.hiascend.com/software/mindstudio/training)。 - 脚本分析工具可以帮助用户在执行迁移操作前，分析基于 GPU 平台的 PyTorch 训练脚本中算子、三方库套件、API 亲和性以及动态 shape 的支持情况。 +![training_process](debug/resources/training_process.png) -2. [（推荐）自动迁移工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E8%87%AA%E5%8A%A8%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - - 自动迁移工具只需在训练脚本中导入库代码即可完成模型脚本的迁移，使用方式简单，且修改内容少。 - -3. [脚本迁移工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E8%84%9A%E6%9C%AC%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - - 脚本迁移工具通过后端命令行，将 GPU 上训练的 PyTorch 脚本迁移至 NPU 上，得到新的训练脚本用于训练。 +# 使用说明 ## [精度工具](./debug/accuracy_tools/) @@ -48,6 +42,10 @@ 绑核脚本，支持非侵入修改工程代码，实现一键式绑核功能。 +5. [msMonitor](./msmonitor) + + MindStudio一站式在线监控工具。 + ## [Tensorboard](./plugins/tensorboard-plugins/tb_plugin) Tensorboard 支持 NPU 性能数据可视化插件 PyTorch Profiler TensorBoard NPU Plugin。 diff --git a/debug/accuracy_tools/cmake/Findgtest.cmake b/debug/accuracy_tools/cmake/Findgtest.cmake index dbfe76abcc9b5d3c2f61642cc8c6e270fc441a0f..d4dd8d8895466d3367dff2032a7de03c829e3dc6 100644 --- a/debug/accuracy_tools/cmake/Findgtest.cmake +++ b/debug/accuracy_tools/cmake/Findgtest.cmake @@ -1,7 +1,6 @@ set(PACKAGE_VERSION 1.12.1) set(PKG_NAME gtest) -set(URL "https://gitee.com/mirrors/googletest/repository/archive/release-1.12.1.tar.gz") set(SHA256_VALUE "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2") set(DOWNLOAD_PATH "$ENV{PROJECT_ROOT_PATH}/third_party") set(DIR_NAME "${DOWNLOAD_PATH}/googletest-release-1.12.1") @@ -9,7 +8,6 @@ set(DIR_NAME "${DOWNLOAD_PATH}/googletest-release-1.12.1") if (NOT ${PKG_NAME}_FOUND) download_opensource_pkg(${PKG_NAME} - URL ${URL} SHA256 ${SHA256_VALUE} DOWNLOAD_PATH ${DOWNLOAD_PATH} ) diff --git a/debug/accuracy_tools/cmake/Findmockcpp.cmake b/debug/accuracy_tools/cmake/Findmockcpp.cmake index c360702c187bfdef553a6b67344ea132a18373f6..73b1729aa5bec968c3e127560db981885c80ba83 100644 --- a/debug/accuracy_tools/cmake/Findmockcpp.cmake +++ b/debug/accuracy_tools/cmake/Findmockcpp.cmake @@ -1,7 +1,6 @@ set(PACKAGE_VERSION 2.7) set(PKG_NAME mockcpp) -set(URL "https://gitee.com/sinojelly/mockcpp/repository/archive/v2.7.zip") set(SHA256_VALUE "0dc7111c5be9785d0550ed3b68db7e12fd5d7802b7bc6548c52ac7b9e727fcc1") set(DOWNLOAD_PATH "$ENV{PROJECT_ROOT_PATH}/third_party") set(DIR_NAME "${DOWNLOAD_PATH}/mockcpp-v2.7") @@ -9,7 +8,6 @@ set(DIR_NAME "${DOWNLOAD_PATH}/mockcpp-v2.7") if (NOT ${PKG_NAME}_FOUND) download_opensource_pkg(${PKG_NAME} - URL ${URL} SHA256 ${SHA256_VALUE} DOWNLOAD_PATH ${DOWNLOAD_PATH} ) diff --git a/debug/accuracy_tools/cmake/Findnlohmannjson.cmake b/debug/accuracy_tools/cmake/Findnlohmannjson.cmake index 0f85cc00a0d30a3896a8f47cac95911929070e33..7acac96ca3ff8025745a6eeddbdf568e453a58f1 100644 --- a/debug/accuracy_tools/cmake/Findnlohmannjson.cmake +++ b/debug/accuracy_tools/cmake/Findnlohmannjson.cmake @@ -1,7 +1,6 @@ set(PACKAGE_VERSION 3.10.1) set(PKG_NAME nlohmannjson) -set(URL "https://gitee.com/mirrors/JSON-for-Modern-CPP/repository/archive/v3.10.1.zip") set(SHA256_VALUE "5c7d0a0542431fef628f8dc4c34fd022fe8747ccb577012d58f38672d8747e0d") set(DOWNLOAD_PATH "$ENV{PROJECT_ROOT_PATH}/third_party") set(DIR_NAME "${DOWNLOAD_PATH}/JSON-for-Modern-CPP-v3.10.1") @@ -9,7 +8,6 @@ set(DIR_NAME "${DOWNLOAD_PATH}/JSON-for-Modern-CPP-v3.10.1") if (NOT ${PKG_NAME}_FOUND) download_opensource_pkg(${PKG_NAME} - URL ${URL} SHA256 ${SHA256_VALUE} DOWNLOAD_PATH ${DOWNLOAD_PATH} ) diff --git a/debug/accuracy_tools/cmake/Findopenssl.cmake b/debug/accuracy_tools/cmake/Findopenssl.cmake index d361095242917df8accbb81a51de65c5ca5ac980..cc33bfc5902aa4c1651029789f04c8a4d2dc10bf 100644 --- a/debug/accuracy_tools/cmake/Findopenssl.cmake +++ b/debug/accuracy_tools/cmake/Findopenssl.cmake @@ -1,7 +1,6 @@ set(PACKAGE_VERSION 1.1.1) set(PKG_NAME openssl) -set(URL "https://gitee.com/mirrors/openssl/repository/archive/OpenSSL_1_1_1k.tar.gz") set(SHA256_VALUE "b92f9d3d12043c02860e5e602e50a73ed21a69947bcc74d391f41148e9f6aa95") set(DOWNLOAD_PATH "$ENV{PROJECT_ROOT_PATH}/third_party") set(DIR_NAME "${DOWNLOAD_PATH}/openssl-OpenSSL_1_1_1k") @@ -23,7 +22,6 @@ endif() endif() download_opensource_pkg(${PKG_NAME} - URL ${URL} SHA256 ${SHA256_VALUE} DOWNLOAD_PATH ${DOWNLOAD_PATH} ) diff --git a/debug/accuracy_tools/cmake/Findprotobuf.cmake b/debug/accuracy_tools/cmake/Findprotobuf.cmake index 4d70515e980f7a921447250fe58400f600419e4c..62c1fe7fbbebc6e0d76fec309a0154d5b102d3aa 100644 --- a/debug/accuracy_tools/cmake/Findprotobuf.cmake +++ b/debug/accuracy_tools/cmake/Findprotobuf.cmake @@ -1,10 +1,9 @@ -set(PACKAGE_VERSION 3.13.0) +set(PACKAGE_VERSION 3.15.0) set(PKG_NAME protobuf) -set(URL "https://gitee.com/mirrors/protobuf_source/repository/archive/v3.13.0.tar.gz") -set(SHA256_VALUE "ab9b39e7053a6fb06b01bf75fb6ec6a71a1ada5a5f8e2446f927336e97b9e7bb") +set(SHA256_VALUE "a1ce078c369f46a3277fdc7ce462ac73cb7cb0edec8bc9d90d23fdb34491c575") set(DOWNLOAD_PATH "$ENV{PROJECT_ROOT_PATH}/third_party") -set(DIR_NAME "${DOWNLOAD_PATH}/protobuf_source-v3.13.0") +set(DIR_NAME "${DOWNLOAD_PATH}/protobuf_source-v3.15.0") if (NOT ${PKG_NAME}_FOUND) @@ -32,7 +31,6 @@ endif() endif() download_opensource_pkg(${PKG_NAME} - URL ${URL} SHA256 ${SHA256_VALUE} DOWNLOAD_PATH ${DOWNLOAD_PATH} ) diff --git a/debug/accuracy_tools/cmake/config.ini b/debug/accuracy_tools/cmake/config.ini new file mode 100644 index 0000000000000000000000000000000000000000..57e544d540aafa1ddf67245d95a78cdc9a151fae --- /dev/null +++ b/debug/accuracy_tools/cmake/config.ini @@ -0,0 +1,14 @@ +[gtest] +url = https://gitee.com/mirrors/googletest/repository/archive/release-1.12.1.tar.gz + +[mockcpp] +url = https://gitee.com/sinojelly/mockcpp/repository/archive/v2.7.zip + +[nlohmannjson] +url = https://gitee.com/mirrors/JSON-for-Modern-CPP/repository/archive/v3.10.1.zip + +[openssl] +url = https://gitee.com/mirrors/openssl/repository/archive/OpenSSL_1_1_1k.tar.gz + +[protobuf] +url = https://gitee.com/mirrors/protobuf_source/repository/archive/v3.15.0.tar.gz \ No newline at end of file diff --git a/debug/accuracy_tools/cmake/download_opensource.sh b/debug/accuracy_tools/cmake/download_opensource.sh index 725e971621434c32d9954c80b9efe234502eefcc..1f815391aa3fdfd60d2e17d499585d351a2a0f04 100644 --- a/debug/accuracy_tools/cmake/download_opensource.sh +++ b/debug/accuracy_tools/cmake/download_opensource.sh @@ -1,11 +1,11 @@ #!/bin/bash if [ "$#" -lt 2 ]; then - echo "Usage: $0 [ ] [ ]" + echo "Usage: $0 [ ] [ ]" exit 1 fi -url=$1 +pkg_name=$1 path=$2 if [ "$#" -ge 3 ]; then @@ -15,6 +15,12 @@ if [ "$#" -ge 4 ]; then tag=$4 fi +url=$(awk -F " = " '/\['${pkg_name}'\]/{a=1}a==1&&$1~/url/{print $2;exit}' config.ini) +if [[ ! $url = https* ]]; then + echo "The URL of $pkg_name is illegal." + exit 1 +fi + echo "Start to download ${url}..." if [ ! -d "$path" ]; then diff --git a/debug/accuracy_tools/cmake/utils.cmake b/debug/accuracy_tools/cmake/utils.cmake index e3e963d63e99da4e0bb1fd2973051278feb04435..738afff874f37bea442c33f6cf607a21bdd6cbe7 100644 --- a/debug/accuracy_tools/cmake/utils.cmake +++ b/debug/accuracy_tools/cmake/utils.cmake @@ -2,13 +2,10 @@ function(download_opensource_pkg pkg_name) message("start to download ${pkg_name}...") set(options) - set(oneValueArgs URL SHA256 GIT_TAG DOWNLOAD_PATH DIR_NAME BUILD_CMD) + set(oneValueArgs SHA256 GIT_TAG DOWNLOAD_PATH DIR_NAME BUILD_CMD) set(multiValueArgs PATCHES) cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - if (NOT PKG_URL) - message(FATAL_ERROR "${pkg_name} need URL.") - endif() if (NOT PKG_DOWNLOAD_PATH) set(PKG_DOWNLOAD_PATH "${CMAKE_SOURCE_DIR}/../third_party") endif() @@ -16,7 +13,7 @@ function(download_opensource_pkg pkg_name) execute_process( WORKING_DIRECTORY $ENV{PROJECT_ROOT_PATH}/cmake - COMMAND bash download_opensource.sh ${PKG_URL} ${PKG_DOWNLOAD_PATH} ${PKG_SHA256} ${PKG_GIT_TAG} + COMMAND bash download_opensource.sh ${pkg_name} ${PKG_DOWNLOAD_PATH} ${PKG_SHA256} ${PKG_GIT_TAG} RESULT_VARIABLE RESULT ) if (NOT RESULT EQUAL 0) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 6b7d483078a6a744ce935591ced0971dea2f5b2f..4951aaf6b6cc1813268481191ee7f14965f7046f 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -54,7 +54,9 @@ export MSPROBE_LOG_LEVEL={x} **2. 工具读写的所有路径，如config_path、dump_path等，只允许包含大小写字母、数字、下划线、斜杠、点和短横线。** -## ⚙️ [安装](./docs/01.installation.md) +## ⚙️ 安装 + +请参见[安装指导说明](./docs/01.installation.md)。 ## 🌟 新版本特性 @@ -138,6 +140,8 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. [PyTorch 单算子API自动生成脚本](./docs/23.generate_operator_PyTorch.md) +[MindSpore 单算子API自动生成脚本](./docs/33.generate_operator_MindSpore.md) + ### 11 数码关联 该功能只支持 MindSpore 静态图场景，用于将IR图与dump数据进行关联，获取dump数据和代码调用栈的关联关系。 @@ -155,6 +159,27 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. [MSAdapter 场景的溢出检测](./docs/30.overflow_check_MSAdapter.md) +### 13 训练检查 + +该工具主要包括： + +训练前或精度比对前，对比两个环境下可能影响训练精度的配置差异。 + +[PyTorch 训练前配置检查](./docs/31.config_check.md) + +训练过程中或结束后，比较两个不同的checkpoint，评估模型相似度。 + +[checkpoint比对](./docs/32.ckpt_compare.md) + +### 14 强化学习数据采集 + +主要能力： + +灵活采集强化学习中重要关键过程数据，并支持比对。 + +[强化学习数据采集](./docs/34.RL_collect.md) + + ## 📑 补充材料 [无标杆比对功能在 PyTorch 场景的性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md) diff --git a/debug/accuracy_tools/msprobe/ccsrc/CMakeLists.txt b/debug/accuracy_tools/msprobe/ccsrc/CMakeLists.txt index 2579a3a0e785c0e0ca384b4d52118a5d828249f8..8472c1ad714f37f045e2c41b7e17ec6f3d709bb6 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/CMakeLists.txt +++ b/debug/accuracy_tools/msprobe/ccsrc/CMakeLists.txt @@ -26,6 +26,8 @@ compile_protobuf_file( ${PROTO_SRC} ) +set(CMAKE_SKIP_RPATH TRUE) + add_library(_msprobe_c SHARED) target_compile_options(_msprobe_c PRIVATE "-Wall") @@ -33,8 +35,9 @@ target_compile_options(_msprobe_c PRIVATE "-fPIC") target_compile_options(_msprobe_c PRIVATE "-fstack-protector-all") target_compile_options(_msprobe_c PRIVATE "-ftrapv") target_compile_options(_msprobe_c PRIVATE "-fstack-check") +target_compile_options(_msprobe_c PRIVATE "-D_FORTIFY_SOURCE=2") -target_link_options(_msprobe_c PRIVATE "-Wl,-z,relor") +target_link_options(_msprobe_c PRIVATE "-Wl,-z,relro") target_link_options(_msprobe_c PRIVATE "-Wl,-z,now") target_link_options(_msprobe_c PRIVATE "-Wl,-z,noexecstack") @@ -50,6 +53,7 @@ if(DEFINED BUILD_TYPE AND "${BUILD_TYPE}" STREQUAL "debug") target_compile_definitions(_msprobe_c PRIVATE __DEBUG__) else() target_compile_options(_msprobe_c PRIVATE "-O2") + target_link_options(_msprobe_c PRIVATE "-s") endif() target_include_directories(_msprobe_c PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.cpp b/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.cpp index 9f61e03a31f6d4dfa2ca0b258d589bbcd29356fa..a23f53f030c924770ca0c635bad933f0db831334 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.cpp @@ -19,18 +19,18 @@ #include #include -#include "include/ErrorCode.hpp" -#include "include/Macro.hpp" -#include "utils/FileUtils.hpp" -#include "base/ErrorInfos.hpp" -#include "DebuggerConfigFieldMap.hpp" -#include "DebuggerConfig.hpp" +#include "include/ErrorCode.h" +#include "include/Macro.h" +#include "utils/FileUtils.h" +#include "base/ErrorInfosManager.h" +#include "DebuggerConfigFieldMap.h" +#include "DebuggerConfig.h" namespace MindStudioDebugger { template DebuggerErrno ParseJsonBaseObj2Var(const nlohmann::json& content, const std::string& field, T& output, - bool mandatory=false) + bool mandatory = false) { nlohmann::json::const_iterator iter = content.find(field); if (iter == content.end()) { @@ -52,7 +52,8 @@ DebuggerErrno ParseJsonBaseObj2Var(const nlohmann::json& content, const std::str template DebuggerErrno ParseJsonStringAndTrans(const nlohmann::json& content, const std::string& field, - const std::map& enum2name, T& output, bool mandatory=false) { + const std::map& enum2name, T& output, bool mandatory = false) +{ DebuggerErrno ret; std::string value; @@ -66,7 +67,7 @@ DebuggerErrno ParseJsonStringAndTrans(const nlohmann::json& content, const std:: } int32_t enumId = GetEnumIdFromName(enum2name, value); - if (enumId == debuggerInvalidEnum) { + if (enumId == DEBUGGER_INVALID_ENUM) { return DebuggerErrno::ERROR_UNKNOWN_VALUE; } @@ -93,19 +94,21 @@ DebuggerErrno ParseJsonStringAndTrans(const nlohmann::json& content, const std:: static bool DebuggerCfgParseUIntRangeGetBorder(const std::string& exp, uint32_t& left, uint32_t& right) { if (std::count(exp.begin(), exp.end(), '-') != 1) { - LOG_ERROR(DebuggerErrno::ERROR_INVALID_FORMAT, "When using a range expression, it should be formatted as \"a-b\"."); + LOG_ERROR(DebuggerErrno::ERROR_INVALID_FORMAT, + "When using a range expression, it should be formatted as \"a-b\"."); return false; } std::istringstream iss(exp); char dash; iss >> left >> dash >> right; if (iss.fail() || dash != '-') { - LOG_ERROR(DebuggerErrno::ERROR_INVALID_FORMAT, "When using a range expression, it should be formatted as \"a-b\"."); + LOG_ERROR(DebuggerErrno::ERROR_INVALID_FORMAT, + "When using a range expression, it should be formatted as \"a-b\"."); return false; } if (left >= right) { LOG_ERROR(DebuggerErrno::ERROR_INVALID_FORMAT, - "When using a range expression, the left border should be smaller than the right."); + "When using a range expression, the left border should be smaller than the right."); return false; } return true; @@ -135,12 +138,18 @@ void DebuggerCfgParseUIntRange(const nlohmann::json& content, const std::string& realLen++; } else if (element.is_string()) { std::string exp = element.get(); - uint32_t begin, end; + uint32_t begin; + uint32_t end; if (!DebuggerCfgParseUIntRangeGetBorder(exp, begin, end)) { LOG_ERROR(DebuggerErrno::ERROR_INVALID_FORMAT, "Failed to parse " + name + "."); return; } - realLen += (end - begin + 1); + uint32_t rangeSize = end - begin; + if (realLen > UINT32_MAX - (rangeSize + 1)) { + LOG_ERROR(DebuggerErrno::ERROR_VALUE_OVERFLOW, name + " size exceeds limit"); + return; + } + realLen += (rangeSize + 1); buf.emplace_back(std::make_pair(begin, end)); } } @@ -148,7 +157,7 @@ void DebuggerCfgParseUIntRange(const nlohmann::json& content, const std::string& constexpr uint32_t maxEleNum = 65536; if (realLen > maxEleNum) { LOG_ERROR(DebuggerErrno::ERROR_INVALID_FORMAT, - "When using a range expression in " + name + ", maximum of 65536 elements can be expressed."); + "When using a range expression in " + name + ", maximum of 65536 elements can be expressed."); return; } @@ -170,9 +179,9 @@ void CommonCfgParseTasks(const nlohmann::json& content, std::vector(content, kTask, taskName, true); + ret = ParseJsonBaseObj2Var(content, TASK, taskName, true); if (ret == DebuggerErrno::ERROR_FIELD_NOT_EXISTS) { - ret = ParseJsonBaseObj2Var>(content, kTasks, taskNameList, true); + ret = ParseJsonBaseObj2Var>(content, TASKS, taskNameList, true); } else { taskNameList.emplace_back(taskName); } @@ -183,8 +192,8 @@ void CommonCfgParseTasks(const nlohmann::json& content, std::vector& expressions) { for (auto& expression : expressions) { size_t len = expression.size(); - if (strncmp(expression.c_str(), kRegexPrefix, kRegexPrefixLen) == 0 && - strncmp(expression.c_str() + (len - kRegexSuffixLen), kRegexSuffix, kRegexSuffixLen) == 0) { - /* name-regex(xxx)表示正则表达式*/ - regexList.emplace_back(expression.substr(kRegexPrefixLen, len - kRegexPrefixLen - kRegexSuffixLen)); + if (strncmp(expression.c_str(), REGEX_PREFIX, REGEX_PREFIX_LEN) == 0 && + strncmp(expression.c_str() + (len - REGEX_SUFFIX_LEN), REGEX_SUFFIX, REGEX_SUFFIX_LEN) == 0) { + /* name-regex(xxx)表示正则表达式 */ + regexList.emplace_back(expression.substr(REGEX_PREFIX_LEN, len - REGEX_PREFIX_LEN - REGEX_SUFFIX_LEN)); } else { /* 否则认为是full scope name */ fullNameList.emplace_back(expression); @@ -219,7 +228,7 @@ std::vector KernelListMatcher::GenRealKernelList(const char** fullK { std::vector output; /* 返回空列表表示全部dump，返回一个空字符串表示没有匹配上的，都不dump */ - if (this->empty() || fullKernelList == nullptr) { + if (this->Empty() || fullKernelList == nullptr) { return output; } output = fullNameList; @@ -247,34 +256,38 @@ void CommonCfg::Parse(const nlohmann::json& content) return; } - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kOutputPath, outputPath); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, OUTPUT_PATH, outputPath); outputPath = FileUtils::GetAbsPath(outputPath); - DebuggerCfgParseUIntRange(content, kRank, rank); - DebuggerCfgParseUIntRange(content, kStep, step); - PARSE_OPTIONAL_FIELD_TRANS_CHECK_RET(content, kLevel, DebuggerLevelEnum2Name, level); - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kSeed, seed); - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kIsDeterministic, isDeterministic); - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kEnableDataloader, enableDataloader); - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kAclConfig, aclConfig); + DebuggerCfgParseUIntRange(content, RANK, rank); + DebuggerCfgParseUIntRange(content, STEP, step); + PARSE_OPTIONAL_FIELD_TRANS_CHECK_RET(content, LEVEL, DEBUGGER_LEVEL_ENUM_2_NAME, level); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, SEED, seed); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, IS_DETERMINISTIC, isDeterministic); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, ENABLE_DATALOADER, enableDataloader); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, ACL_CONFIG, aclConfig); } void DebuggerCfgParseDataMode(const nlohmann::json& content, DebuggerDataDirection& direction, DebuggerDataInOut& inout) { std::vector buf; - bool fw, bw, in, out, all; + bool fw; + bool bw; + bool in; + bool out; + bool all; direction = DebuggerDataDirection::DIRECTION_BOTH; inout = DebuggerDataInOut::INOUT_BOTH; - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kDataMode, buf); - all = static_cast(std::find(buf.begin(), buf.end(), kDataModeAll) != buf.end()); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, DATA_MODE, buf); + all = static_cast(std::find(buf.begin(), buf.end(), DATA_MODE_ALL) != buf.end()); if (buf.empty() || all) { return; } - fw = static_cast(std::find(buf.begin(), buf.end(), kDirectionForward) != buf.end()); - bw = static_cast(std::find(buf.begin(), buf.end(), kDirectionBackward) != buf.end()); - in = static_cast(std::find(buf.begin(), buf.end(), kInOutInput) != buf.end()); - out = static_cast(std::find(buf.begin(), buf.end(), kInOutOutput) != buf.end()); + fw = static_cast(std::find(buf.begin(), buf.end(), DIRECTION_FORWARD) != buf.end()); + bw = static_cast(std::find(buf.begin(), buf.end(), DIRECTION_BACKWARD) != buf.end()); + in = static_cast(std::find(buf.begin(), buf.end(), INOUT_INPUT) != buf.end()); + out = static_cast(std::find(buf.begin(), buf.end(), INOUT_OUTPUT) != buf.end()); /* 互补项都配或都不配都表示both，因此关注不同的场景就行 */ if (fw != bw) { @@ -298,18 +311,18 @@ void StatisticsCfgParseSummary(const nlohmann::json& content, std::vector modeListName; /* 若无该字段，认为是statistic，因此这里给mode设个默认值 */ - ret = ParseJsonBaseObj2Var(content, kSummaryMode, mode); + ret = ParseJsonBaseObj2Var(content, SUMMARY_MODE, mode); if (ret == DebuggerErrno::OK) { - if (mode == kStatistics) { + if (mode == STATISTICS) { summaryOption.push_back(DebuggerSummaryOption::MAX); summaryOption.push_back(DebuggerSummaryOption::MIN); summaryOption.push_back(DebuggerSummaryOption::MEAN); summaryOption.push_back(DebuggerSummaryOption::L2NORM); - } else if (mode == kMd5) { + } else if (mode == MD5) { summaryOption.push_back(DebuggerSummaryOption::MD5); } else { LOG_ERROR(DebuggerErrno::ERROR_UNKNOWN_VALUE, "Summary mode " + mode + " is unknown."); @@ -317,7 +330,7 @@ void StatisticsCfgParseSummary(const nlohmann::json& content, std::vector>(content, kSummaryMode, modeListName); + ret = ParseJsonBaseObj2Var>(content, SUMMARY_MODE, modeListName); if (ret != DebuggerErrno::OK) { LOG_ERROR(ret, "Value of field summary_mode should be string or list."); return; @@ -333,8 +346,8 @@ void StatisticsCfgParseSummary(const nlohmann::json& content, std::vector filter; - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kScope, scope); - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kList, filter); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, SCOPE, scope); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, LIST, filter); filter.erase(std::remove_if(filter.begin(), filter.end(), [](const std::string& s) { return s.find_first_not_of(' ') == std::string::npos; }), - filter.end()); + filter.end()); list = std::move(filter); if (DebuggerConfig::GetInstance().GetDebugLevel() == DebuggerLevel::L2) { matcher.Parse(list); @@ -363,24 +376,24 @@ void StatisticsCfg::Parse(const nlohmann::json& content) void DumpTensorCfg::Parse(const nlohmann::json& content) { std::vector filter; - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kScope, scope); - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kList, filter); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, SCOPE, scope); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, LIST, filter); filter.erase(std::remove_if(filter.begin(), filter.end(), [](const std::string& s) { return s.find_first_not_of(' ') == std::string::npos; }), - filter.end()); + filter.end()); list = std::move(filter); if (DebuggerConfig::GetInstance().GetDebugLevel() == DebuggerLevel::L2) { matcher.Parse(list); } DebuggerCfgParseDataMode(content, direction, inout); - PARSE_OPTIONAL_FIELD_TRANS_CHECK_RET(content, kFileFormat, DumpFileFormatEnum2Name, fileFormat); - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kBackwardInput, backwardInput); + PARSE_OPTIONAL_FIELD_TRANS_CHECK_RET(content, FILE_FORMAT, DUMP_FILE_FORMAT_ENUM_2_NAME, fileFormat); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, BACKWARD_INPUT, backwardInput); } void OverflowCheckCfg::Parse(const nlohmann::json& content) { - PARSE_OPTIONAL_FIELD_CHECK_RET(content, kOverflowNums, overflowNums); - PARSE_OPTIONAL_FIELD_TRANS_CHECK_RET(content, kCheckMode, OpCheckLevelEnum2Name, checkMode); + PARSE_OPTIONAL_FIELD_CHECK_RET(content, OVERFLOW_NUMS, overflowNums); + PARSE_OPTIONAL_FIELD_TRANS_CHECK_RET(content, CHECK_MODE, OP_CHECK_LEVEL_ENUM_2_NAME, checkMode); } void DebuggerConfig::Reset() @@ -419,14 +432,14 @@ void DebuggerConfig::Parse() iter = content.find(name); \ if (iter != content.end()) { \ member = std::make_shared(); \ - member->Parse(*(iter)); \ + ((member)->Parse(*(iter))); \ } \ } \ } while (0) - PARSE_SUBTASK_CONFIG(DebuggerTaskType::TASK_DUMP_STATISTICS, kTaskStatistics, statisticCfg, StatisticsCfg); - PARSE_SUBTASK_CONFIG(DebuggerTaskType::TASK_DUMP_TENSOR, kTaskDumpTensor, dumpTensorCfg, DumpTensorCfg); - PARSE_SUBTASK_CONFIG(DebuggerTaskType::TASK_OVERFLOW_CHECK, kTaskOverflowCheck, overflowCheckCfg, OverflowCheckCfg); + PARSE_SUBTASK_CONFIG(DebuggerTaskType::TASK_DUMP_STATISTICS, TASK_STATISTICS, statisticCfg, StatisticsCfg); + PARSE_SUBTASK_CONFIG(DebuggerTaskType::TASK_DUMP_TENSOR, TASK_DUMP_TENSOR, dumpTensorCfg, DumpTensorCfg); + PARSE_SUBTASK_CONFIG(DebuggerTaskType::TASK_OVERFLOW_CHECK, TASK_OVERFLOW_CHECK, overflowCheckCfg, OverflowCheckCfg); #undef PARSE_SUBTASK_CONFIG return; @@ -451,8 +464,8 @@ int32_t DebuggerConfig::LoadConfig(const std::string& framework, const std::stri return -1; } - int32_t enumId = GetEnumIdFromName(FrameworkEnum2Name, framework); - if (enumId == debuggerInvalidEnum) { + int32_t enumId = GetEnumIdFromName(FRAMEWORK_ENUM_2_NAME, framework); + if (enumId == DEBUGGER_INVALID_ENUM) { LOG_ERROR(DebuggerErrno::ERROR_UNKNOWN_VALUE, "Unknown framework " + framework + "."); return -1; } diff --git a/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.hpp b/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.h similarity index 91% rename from debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.hpp rename to debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.h index d56191443f8e6a7819c2bfbf402a5937bacd92ff..e9390ffe461e2586b518384ed0650fb18453b6c0 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfig.h @@ -26,11 +26,11 @@ #include #include -#include "include/Macro.hpp" +#include "include/Macro.h" namespace MindStudioDebugger { -constexpr int debuggerInvalidEnum = -1; +constexpr int DEBUGGER_INVALID_ENUM = -1; enum class DebuggerFramework { FRAMEWORK_PYTORCH, @@ -47,7 +47,7 @@ enum class DebuggerTaskType { TASK_RUN_UT, TASK_GRAD_PROBE, - TASK_BUTT = debuggerInvalidEnum, + TASK_BUTT = DEBUGGER_INVALID_ENUM, }; enum class DebuggerDevType { @@ -55,7 +55,7 @@ enum class DebuggerDevType { DEVICE_TYPE_GPU, DEVICE_TYPE_CPU, - DEVICE_TYPE_BUTT = debuggerInvalidEnum, + DEVICE_TYPE_BUTT = DEBUGGER_INVALID_ENUM, }; enum class DebuggerLevel { @@ -64,7 +64,7 @@ enum class DebuggerLevel { L2, MIX, - LEVEL_BUTT = debuggerInvalidEnum, + LEVEL_BUTT = DEBUGGER_INVALID_ENUM, }; enum class DebuggerDataDirection { @@ -72,7 +72,7 @@ enum class DebuggerDataDirection { DIRECTION_BACKWARD, DIRECTION_BOTH, - DIRECTION_BUTT = debuggerInvalidEnum, + DIRECTION_BUTT = DEBUGGER_INVALID_ENUM, }; enum class DebuggerDataInOut { @@ -80,14 +80,14 @@ enum class DebuggerDataInOut { INOUT_OUTPUT, INOUT_BOTH, - INOUT_BUTT = debuggerInvalidEnum, + INOUT_BUTT = DEBUGGER_INVALID_ENUM, }; enum class DebuggerDumpFileFormat { FILE_FORMAT_BIN, FILE_FORMAT_NPY, - FILE_FORMAT_BUTT = debuggerInvalidEnum, + FILE_FORMAT_BUTT = DEBUGGER_INVALID_ENUM, }; enum class DebuggerOpCheckLevel { @@ -95,7 +95,7 @@ enum class DebuggerOpCheckLevel { CHECK_LEVEL_ATOMIC, CHECK_LEVEL_ALL, - CHECK_LEVEL_BUTT = debuggerInvalidEnum, + CHECK_LEVEL_BUTT = DEBUGGER_INVALID_ENUM, }; enum class DebuggerSummaryOption { @@ -108,7 +108,7 @@ enum class DebuggerSummaryOption { POS_INF_CNT, MD5, - SUMMARY_BUTT = debuggerInvalidEnum, + SUMMARY_BUTT = DEBUGGER_INVALID_ENUM, }; class KernelListMatcher { @@ -119,8 +119,8 @@ public: void Parse(const std::vector& expressions); std::vector GenRealKernelList(const char** fullKernelList) const; - inline bool empty() const {return fullNameList.empty() && regexList.empty();} - inline bool needAllKernels() const {return !regexList.empty();} + inline bool Empty() const {return fullNameList.empty() && regexList.empty();} + inline bool NeedAllKernels() const {return !regexList.empty();} private: std::vector fullNameList; @@ -208,11 +208,11 @@ private: class DebuggerConfig { - public: - static DebuggerConfig& GetInstance() { - static DebuggerConfig instance_; - return instance_; + static DebuggerConfig& GetInstance() + { + static DebuggerConfig configInstance; + return configInstance; } int32_t LoadConfig(const std::string& framework, const std::string& cfgFilePath); diff --git a/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfigFieldMap.hpp b/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfigFieldMap.h similarity index 30% rename from debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfigFieldMap.hpp rename to debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfigFieldMap.h index 8ebef4206b42b702712edccc5b19d9611370c63b..95954ecd417275c6e38fc37f01a6f8bb18c939e4 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfigFieldMap.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/base/DebuggerConfigFieldMap.h @@ -19,129 +19,129 @@ #include #include -#include "DebuggerConfig.hpp" +#include "DebuggerConfig.h" namespace MindStudioDebugger { -constexpr const char* kFramework = "framework"; -constexpr const char* kFrameworkPyTorch = "PyTorch"; -constexpr const char* kFrameworkMindSpore = "MindSpore"; - -constexpr const char* kTaskStatistics = "statistics"; -constexpr const char* kTaskDumpTensor = "tensor"; -constexpr const char* kTaskOverflowCheck = "overflow_check"; -constexpr const char* kFreeBenchmark = "free_benchmark"; -constexpr const char* kRunUT = "run_ut"; -constexpr const char* kGradProbe = "grad_probe"; - -constexpr const char* kLevel0 = "L0"; -constexpr const char* kLevel1 = "L1"; -constexpr const char* kLevel2 = "L2"; -constexpr const char* kLevelMix = "mix"; - -constexpr const char* kDirectionForward = "forward"; -constexpr const char* kDirectionBackward = "backward"; -constexpr const char* kDirectionBoth = "both"; -constexpr const char* kInOutInput = "input"; -constexpr const char* kInOutOutput = "output"; -constexpr const char* kInOutBoth = "both"; -constexpr const char* kDataModeAll = "all"; - -constexpr const char* kFreeBenchmarkHandlerCheck = "check"; -constexpr const char* kFreeBenchmarkHandlerFix = "fix"; - -constexpr const char* kDumpFileFormatBin = "bin"; -constexpr const char* kDumpFileFormatNpy = "npy"; - -constexpr const char* kOpCheckLevelAiCore = "aicore"; -constexpr const char* kOpCheckLevelAtomic = "atomic"; -constexpr const char* kOpCheckLevelAll = "all"; - -constexpr const char* kTask = "task"; -constexpr const char* kTasks = "tasks"; -constexpr const char* kOutputPath = "dump_path"; -constexpr const char* kRank = "rank"; -constexpr const char* kStep = "step"; -constexpr const char* kLevel = "level"; -constexpr const char* kSeed = "seed"; -constexpr const char* kIsDeterministic = "is_deterministic"; -constexpr const char* kEnableDataloader = "enable_dataloader"; -constexpr const char* kAclConfig = "acl_config"; - -constexpr const char* kScope = "scope"; -constexpr const char* kList = "list"; - -constexpr const char* kDataMode = "data_mode"; -constexpr const char* kSummaryMode = "summary_mode"; -constexpr const char* kFileFormat = "file_format"; -constexpr const char* kOverflowNums = "overflow_nums"; -constexpr const char* kCheckMode = "check_mode"; -constexpr const char* kBackwardInput = "backward_input"; - -constexpr const char* kStatistics = "statistics"; -constexpr const char* kMd5 = "md5"; -constexpr const char* kMax = "max"; -constexpr const char* kMin = "min"; -constexpr const char* kMean = "mean"; -constexpr const char* kL2Norm = "l2norm"; -constexpr const char* kNanCount = "nan count"; -constexpr const char* kNegativeInfCount = "negative inf count"; -constexpr const char* kPositiveInfCount = "positive inf count"; - -const std::map FrameworkEnum2Name = { - {static_cast(DebuggerFramework::FRAMEWORK_PYTORCH), kFrameworkPyTorch}, - {static_cast(DebuggerFramework::FRAMEWORK_MINDSPORE), kFrameworkMindSpore}, +constexpr const char* FRAMEWORK = "framework"; +constexpr const char* FRAMEWORK_PYTORCH = "PyTorch"; +constexpr const char* FRAMEWORK_MINDSPORE = "MindSpore"; + +constexpr const char* TASK_STATISTICS = "statistics"; +constexpr const char* TASK_DUMP_TENSOR = "tensor"; +constexpr const char* TASK_OVERFLOW_CHECK = "overflow_check"; +constexpr const char* TASK_FREE_BENCHMARK = "free_benchmark"; +constexpr const char* TASK_RUN_UT = "run_ut"; +constexpr const char* TASK_GRAD_PROBE = "grad_probe"; + +constexpr const char* LEVEL0 = "L0"; +constexpr const char* LEVEL1 = "L1"; +constexpr const char* LEVEL2 = "L2"; +constexpr const char* LEVEL_MIX = "mix"; + +constexpr const char* DIRECTION_FORWARD = "forward"; +constexpr const char* DIRECTION_BACKWARD = "backward"; +constexpr const char* DIRECTION_BOTH = "both"; +constexpr const char* INOUT_INPUT = "input"; +constexpr const char* INOUT_OUTPUT = "output"; +constexpr const char* INOUT_BOTH = "both"; +constexpr const char* DATA_MODE_ALL = "all"; + +constexpr const char* FREE_BENCHMARK_HANDLER_CHECK = "check"; +constexpr const char* FREE_BENCHMARK_HANDLER_FIX = "fix"; + +constexpr const char* DUMP_FILE_FORMAT_BIN = "bin"; +constexpr const char* DUMP_FILE_FORMAT_NPY = "npy"; + +constexpr const char* OP_CHECK_LEVEL_AICORE = "aicore"; +constexpr const char* OP_CHECK_LEVEL_ATOMIC = "atomic"; +constexpr const char* OP_CHECK_LEVEL_ALL = "all"; + +constexpr const char* TASK = "task"; +constexpr const char* TASKS = "tasks"; +constexpr const char* OUTPUT_PATH = "dump_path"; +constexpr const char* RANK = "rank"; +constexpr const char* STEP = "step"; +constexpr const char* LEVEL = "level"; +constexpr const char* SEED = "seed"; +constexpr const char* IS_DETERMINISTIC = "is_deterministic"; +constexpr const char* ENABLE_DATALOADER = "enable_dataloader"; +constexpr const char* ACL_CONFIG = "acl_config"; + +constexpr const char* SCOPE = "scope"; +constexpr const char* LIST = "list"; + +constexpr const char* DATA_MODE = "data_mode"; +constexpr const char* SUMMARY_MODE = "summary_mode"; +constexpr const char* FILE_FORMAT = "file_format"; +constexpr const char* OVERFLOW_NUMS = "overflow_nums"; +constexpr const char* CHECK_MODE = "check_mode"; +constexpr const char* BACKWARD_INPUT = "backward_input"; + +constexpr const char* STATISTICS = "statistics"; +constexpr const char* MD5 = "md5"; +constexpr const char* MAX = "max"; +constexpr const char* MIN = "min"; +constexpr const char* MEAN = "mean"; +constexpr const char* L2_NORM = "l2norm"; +constexpr const char* NAN_COUNT = "nan count"; +constexpr const char* NEGATIVE_INF_COUNT = "negative inf count"; +constexpr const char* POSITIVE_INF_COUNT = "positive inf count"; + +const std::map FRAMEWORK_ENUM_2_NAME = { + {static_cast(DebuggerFramework::FRAMEWORK_PYTORCH), FRAMEWORK_PYTORCH}, + {static_cast(DebuggerFramework::FRAMEWORK_MINDSPORE), FRAMEWORK_MINDSPORE}, }; -const std::map TaskTypeEnum2Name = { - {static_cast(DebuggerTaskType::TASK_DUMP_TENSOR), kTaskDumpTensor}, - {static_cast(DebuggerTaskType::TASK_DUMP_STATISTICS), kTaskStatistics}, - {static_cast(DebuggerTaskType::TASK_OVERFLOW_CHECK), kTaskOverflowCheck}, - {static_cast(DebuggerTaskType::TASK_FREE_BENCHMARK), kFreeBenchmark}, - {static_cast(DebuggerTaskType::TASK_RUN_UT), kRunUT}, - {static_cast(DebuggerTaskType::TASK_GRAD_PROBE), kGradProbe}, +const std::map TASK_TYPE_ENUM_2_NAME = { + {static_cast(DebuggerTaskType::TASK_DUMP_TENSOR), TASK_DUMP_TENSOR}, + {static_cast(DebuggerTaskType::TASK_DUMP_STATISTICS), TASK_STATISTICS}, + {static_cast(DebuggerTaskType::TASK_OVERFLOW_CHECK), TASK_OVERFLOW_CHECK}, + {static_cast(DebuggerTaskType::TASK_FREE_BENCHMARK), TASK_FREE_BENCHMARK}, + {static_cast(DebuggerTaskType::TASK_RUN_UT), TASK_RUN_UT}, + {static_cast(DebuggerTaskType::TASK_GRAD_PROBE), TASK_GRAD_PROBE}, }; -const std::map DebuggerLevelEnum2Name = { - {static_cast(DebuggerLevel::L0), kLevel0}, - {static_cast(DebuggerLevel::L1), kLevel1}, - {static_cast(DebuggerLevel::L2), kLevel2}, - {static_cast(DebuggerLevel::MIX), kLevelMix}, +const std::map DEBUGGER_LEVEL_ENUM_2_NAME = { + {static_cast(DebuggerLevel::L0), LEVEL0}, + {static_cast(DebuggerLevel::L1), LEVEL0}, + {static_cast(DebuggerLevel::L2), LEVEL2}, + {static_cast(DebuggerLevel::MIX), LEVEL_MIX}, }; -const std::map DataDirectionEnum2Name = { - {static_cast(DebuggerDataDirection::DIRECTION_FORWARD), kDirectionForward}, - {static_cast(DebuggerDataDirection::DIRECTION_BACKWARD), kDirectionBackward}, - {static_cast(DebuggerDataDirection::DIRECTION_BOTH), kDirectionBoth}, +const std::map DATA_DIRECTION_ENUM_2_NAME = { + {static_cast(DebuggerDataDirection::DIRECTION_FORWARD), DIRECTION_FORWARD}, + {static_cast(DebuggerDataDirection::DIRECTION_BACKWARD), DIRECTION_BACKWARD}, + {static_cast(DebuggerDataDirection::DIRECTION_BOTH), DIRECTION_BOTH}, }; -const std::map DataInOutEnum2Name = { - {static_cast(DebuggerDataInOut::INOUT_INPUT), kInOutInput}, - {static_cast(DebuggerDataInOut::INOUT_OUTPUT), kInOutOutput}, - {static_cast(DebuggerDataInOut::INOUT_BOTH), kInOutBoth}, +const std::map DATA_INOUT_ENUM_2_NAME = { + {static_cast(DebuggerDataInOut::INOUT_INPUT), INOUT_INPUT}, + {static_cast(DebuggerDataInOut::INOUT_OUTPUT), INOUT_OUTPUT}, + {static_cast(DebuggerDataInOut::INOUT_BOTH), INOUT_BOTH}, }; -const std::map DumpFileFormatEnum2Name = { - {static_cast(DebuggerDumpFileFormat::FILE_FORMAT_BIN), kDumpFileFormatBin}, - {static_cast(DebuggerDumpFileFormat::FILE_FORMAT_NPY), kDumpFileFormatNpy}, +const std::map DUMP_FILE_FORMAT_ENUM_2_NAME = { + {static_cast(DebuggerDumpFileFormat::FILE_FORMAT_BIN), DUMP_FILE_FORMAT_BIN}, + {static_cast(DebuggerDumpFileFormat::FILE_FORMAT_NPY), DUMP_FILE_FORMAT_NPY}, }; -const std::map OpCheckLevelEnum2Name = { - {static_cast(DebuggerOpCheckLevel::CHECK_LEVEL_AICORE), kOpCheckLevelAiCore}, - {static_cast(DebuggerOpCheckLevel::CHECK_LEVEL_ATOMIC), kOpCheckLevelAtomic}, - {static_cast(DebuggerOpCheckLevel::CHECK_LEVEL_ALL), kOpCheckLevelAll}, +const std::map OP_CHECK_LEVEL_ENUM_2_NAME = { + {static_cast(DebuggerOpCheckLevel::CHECK_LEVEL_AICORE), OP_CHECK_LEVEL_AICORE}, + {static_cast(DebuggerOpCheckLevel::CHECK_LEVEL_ATOMIC), OP_CHECK_LEVEL_ATOMIC}, + {static_cast(DebuggerOpCheckLevel::CHECK_LEVEL_ALL), OP_CHECK_LEVEL_ALL}, }; -const std::map SummaryOptionEnum2Name = { - {static_cast(DebuggerSummaryOption::MAX), kMax}, - {static_cast(DebuggerSummaryOption::MIN), kMin}, - {static_cast(DebuggerSummaryOption::MEAN), kMean}, - {static_cast(DebuggerSummaryOption::NAN_CNT), kNanCount}, - {static_cast(DebuggerSummaryOption::NEG_INF_CNT), kNegativeInfCount}, - {static_cast(DebuggerSummaryOption::POS_INF_CNT), kPositiveInfCount}, - {static_cast(DebuggerSummaryOption::L2NORM), kL2Norm}, +const std::map SUMMARY_OPTION_ENUM_2_NAME = { + {static_cast(DebuggerSummaryOption::MAX), MAX}, + {static_cast(DebuggerSummaryOption::MIN), MIN}, + {static_cast(DebuggerSummaryOption::MEAN), MEAN}, + {static_cast(DebuggerSummaryOption::NAN_CNT), NAN_COUNT}, + {static_cast(DebuggerSummaryOption::NEG_INF_CNT), NEGATIVE_INF_COUNT}, + {static_cast(DebuggerSummaryOption::POS_INF_CNT), POSITIVE_INF_COUNT}, + {static_cast(DebuggerSummaryOption::L2NORM), L2_NORM}, - {static_cast(DebuggerSummaryOption::MD5), kMd5}, + {static_cast(DebuggerSummaryOption::MD5), MD5}, }; inline int32_t GetEnumIdFromName(const std::map& enum2name, const std::string& name) @@ -151,7 +151,7 @@ inline int32_t GetEnumIdFromName(const std::map& enum2name return iter->first; } } - return debuggerInvalidEnum; + return DEBUGGER_INVALID_ENUM; } inline std::string GetNameFromEnumId(const std::map& enum2name, int32_t id) diff --git a/debug/accuracy_tools/msprobe/ccsrc/base/Environment.cpp b/debug/accuracy_tools/msprobe/ccsrc/base/Environment.cpp index 3a31e03cf898901767e3c658b993edc14b76e35a..cfc4c4b164ccdbae3a7cf4173d64a1b180c8b87a 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/base/Environment.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/base/Environment.cpp @@ -14,14 +14,14 @@ * limitations under the License. */ -#include "utils/CPythonUtils.hpp" -#include "DebuggerConfig.hpp" -#include "Environment.hpp" +#include "utils/CPythonUtils.h" +#include "DebuggerConfig.h" +#include "Environment.h" namespace MindStudioDebugger { namespace Environment { -static int32_t GetRankID_PT() +static int32_t GetPTRankID() { /* if torch.distributed.is_initialized(): * return torch.distributed.get_rank() @@ -48,10 +48,10 @@ static int32_t GetRankID_PT() return id; } -static int32_t GetRankID_MS() +static int32_t GetMSRankID() { - constexpr const char* kRankId = "RANK_ID"; - const char* rankIdEnv = getenv(kRankId); + constexpr const char* RANK_ID = "RANK_ID"; + const char* rankIdEnv = getenv(RANK_ID); if (rankIdEnv == nullptr) { return -1; } @@ -78,9 +78,9 @@ int32_t GetRankID() } if (DebuggerConfig::GetInstance().GetFramework() == DebuggerFramework::FRAMEWORK_PYTORCH) { - id = GetRankID_PT(); + id = GetPTRankID(); } else if (DebuggerConfig::GetInstance().GetFramework() == DebuggerFramework::FRAMEWORK_MINDSPORE) { - id = GetRankID_MS(); + id = GetMSRankID(); } return id; diff --git a/debug/accuracy_tools/msprobe/ccsrc/base/Environment.hpp b/debug/accuracy_tools/msprobe/ccsrc/base/Environment.h similarity index 100% rename from debug/accuracy_tools/msprobe/ccsrc/base/Environment.hpp rename to debug/accuracy_tools/msprobe/ccsrc/base/Environment.h diff --git a/debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfos.cpp b/debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfosManager.cpp similarity index 89% rename from debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfos.cpp rename to debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfosManager.cpp index b07554a9fe10609ab4fa03357877b2f7630bd55e..755be22eac060c150aa9bdd508888ae2879a5d90 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfos.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfosManager.cpp @@ -22,13 +22,12 @@ #include #include -#include "utils/FileUtils.hpp" -#include "ErrorInfos.hpp" +#include "utils/FileUtils.h" +#include "ErrorInfosManager.h" namespace MindStudioDebugger { -static std::mutex errInfoMtx; -static std::ofstream logOfs; +static std::mutex g_errInfoMtx; DebuggerErrLevel ErrorInfosManager::topLevel = DebuggerErrLevel::LEVEL_NONE; DebuggerErrLevel ErrorInfosManager::threshold = DebuggerErrLevel::LEVEL_INFO; @@ -84,8 +83,8 @@ void ErrorInfosManager::LogErrorInfo(DebuggerErrLevel level, DebuggerErrno errId return; } - std::lock_guard lk(errInfoMtx); - std::ostream& output = logOfs.is_open() ? logOfs : std::cout; + std::lock_guard lk(g_errInfoMtx); + std::ostream& output = std::cout; output << "[" << ErrorLevelString[level] << "]"; if (errId != DebuggerErrno::NONE) { output << "[" << ErrnoString[errId] << "]"; @@ -101,26 +100,12 @@ void ErrorInfosManager::LogErrorInfo(DebuggerErrLevel level, DebuggerErrno errId DebuggerErrLevel ErrorInfosManager::GetTopErrLevelInDuration() { - std::lock_guard lk(errInfoMtx); + std::lock_guard lk(g_errInfoMtx); DebuggerErrLevel ret = topLevel; topLevel = DebuggerErrLevel::LEVEL_NONE; return ret; } -void ErrorInfosManager::SetLogPath(const std::string& path) -{ - std::lock_guard lk(errInfoMtx); - if (logOfs.is_open()) { - logOfs.close(); - } - - if (path.empty()) { - return; - } - - FileUtils::OpenFile(path, logOfs); -} - __attribute__((constructor)) void InitDebuggerThreshold() { const char* msprobeLogLevelEnv = getenv("MSPROBE_LOG_LEVEL"); diff --git a/debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfos.hpp b/debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfosManager.h similarity index 96% rename from debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfos.hpp rename to debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfosManager.h index 6c740a6a36cfd7692b793dfa7625789771731289..62d1a1e8902da59ebeef90e7c1fd2dd4ce188f21 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfos.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/base/ErrorInfosManager.h @@ -18,7 +18,7 @@ #include #include -#include "include/ErrorCode.hpp" +#include "include/ErrorCode.h" namespace MindStudioDebugger { @@ -35,14 +35,14 @@ class ErrorInfosManager { public: static void LogErrorInfo(DebuggerErrLevel level, DebuggerErrno errId, const std::string& info); static DebuggerErrLevel GetTopErrLevelInDuration(); - static void SetLogPath(const std::string& path); static void SetLogThreshold(DebuggerErrLevel t) { threshold = t; } private: static DebuggerErrLevel topLevel; static DebuggerErrLevel threshold; }; -inline void CleanErrorInfoCache() { +inline void CleanErrorInfoCache() +{ ErrorInfosManager::GetTopErrLevelInDuration(); } diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.cpp b/debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.cpp index d26b1a6a2c341e0a60f0bc71b021f64ab6da5a1b..aa33fee61dd4c06f75c7d723491ec9e32ace812a 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.cpp @@ -25,56 +25,56 @@ #include #include -#include "include/Macro.hpp" -#include "utils/FileUtils.hpp" -#include "utils/FileOperation.hpp" -#include "utils/DataUtils.hpp" -#include "utils/MathUtils.hpp" -#include "core/AclTensor.hpp" -#include "base/ErrorInfos.hpp" +#include "include/Macro.h" +#include "utils/FileUtils.h" +#include "utils/FileOperation.h" +#include "utils/DataUtils.h" +#include "utils/MathUtils.h" +#include "core/AclTensor.h" +#include "base/ErrorInfosManager.h" #include "proto/AclDumpMsg.pb.h" -#include "AclDumpDataProcessor.hpp" +#include "AclDumpDataProcessor.h" namespace MindStudioDebugger { namespace AclDumpMsg = toolkit::dumpdata; -constexpr size_t kDhaAtomicAddInfoSize = 128; -constexpr size_t kL2AtomicAddInfoSize = 128; -constexpr size_t kAiCoreInfoSize = 256; -constexpr size_t kDhaAtomicAddStatusSize = 256; -constexpr size_t kL2AtomicAddStatusSize = 256; -constexpr size_t kUint64Size = sizeof(uint64_t); -constexpr const char* debugFileSign = "Opdebug.Node_OpDebug."; - -constexpr const char* kStatsHeaderInout = "Input/Output"; -constexpr const char* kStatsHeaderId = "Index"; -constexpr const char* kStatsHeaderDataSize = "Data Size"; -constexpr const char* kStatsHeaderDataType = "Data Type"; -constexpr const char* kStatsHeaderFormat = "Format"; -constexpr const char* kStatsHeaderShape = "Shape"; -constexpr const char* kStatsHeaderMax = "Max Value"; -constexpr const char* kStatsHeaderMin = "Min Value"; -constexpr const char* kStatsHeaderAvg = "Avg Value"; -constexpr const char* kStatsHeaderL2Norm = "l2norm"; -constexpr const char* kStatsHeaderL2NormInCsv = "L2Norm Value"; -constexpr const char* kStatsHeaderMD5 = "MD5 Value"; -constexpr const char* kStatsHeaderNan = "Nan Count"; -constexpr const char* kStatsHeaderNanInCsv = "NaN Count"; -constexpr const char* kStatsHeaderNegInf = "Negative Inf Count"; -constexpr const char* kStatsHeaderPosInf = "Positive Inf Count"; -constexpr const char* kRankId = "RANK_ID"; -constexpr const char* kDigitalNumbers = "0123456789"; - -static const std::map> summaryOptionHeaderStrMap = { - {DebuggerSummaryOption::MAX, {kStatsHeaderMax, kStatsHeaderMax}}, - {DebuggerSummaryOption::MIN, {kStatsHeaderMin, kStatsHeaderMin}}, - {DebuggerSummaryOption::MEAN, {kStatsHeaderAvg, kStatsHeaderAvg}}, - {DebuggerSummaryOption::L2NORM, {kStatsHeaderL2Norm, kStatsHeaderL2NormInCsv}}, - {DebuggerSummaryOption::NAN_CNT, {kStatsHeaderNan, kStatsHeaderNanInCsv}}, - {DebuggerSummaryOption::NEG_INF_CNT, {kStatsHeaderNegInf, kStatsHeaderNegInf}}, - {DebuggerSummaryOption::POS_INF_CNT, {kStatsHeaderPosInf, kStatsHeaderPosInf}}, - {DebuggerSummaryOption::MD5, {kStatsHeaderMD5, kStatsHeaderMD5}}, +constexpr size_t DHA_ATOMIC_ADD_INFO_SIZE = 128; +constexpr size_t L2_ATOMIC_ADD_INFO_SIZE = 128; +constexpr size_t AICORE_INFO_SIZE = 256; +constexpr size_t DHA_ATOMIC_ADD_STATUS_SIZE = 256; +constexpr size_t L2_ATOMIC_ADD_STATUS_SIZE = 256; +constexpr size_t UINT64_SIZE = sizeof(uint64_t); +constexpr const char* DEBUG_FILE_SIGN = "Opdebug.Node_OpDebug."; + +constexpr const char* STATS_HEADER_INOUT = "Input/Output"; +constexpr const char* STATS_HEADER_ID = "Index"; +constexpr const char* STATS_HEADER_DATA_SIZE = "Data Size"; +constexpr const char* STATS_HEADER_DATA_TYPE = "Data Type"; +constexpr const char* STATS_HEADER_FORMAT = "Format"; +constexpr const char* STATS_HEADER_SHAPE = "Shape"; +constexpr const char* STATS_HEADER_MAX = "Max Value"; +constexpr const char* STATS_HEADER_MIN = "Min Value"; +constexpr const char* STATS_HEADER_AVG = "Avg Value"; +constexpr const char* STATS_HEADER_L2NORM = "l2norm"; +constexpr const char* STATS_CSV_HEADER_L2NORM = "L2Norm Value"; +constexpr const char* STATS_HEADER_MD5 = "MD5 Value"; +constexpr const char* STATS_HEADER_NAN = "Nan Count"; +constexpr const char* STATS_CSV_HEADER_NAN = "NaN Count"; +constexpr const char* STATS_HEADER_NEG_INF = "Negative Inf Count"; +constexpr const char* STATS_HEADER_POS_INF = "Positive Inf Count"; +constexpr const char* RANK_ID = "RANK_ID"; +constexpr const char* DIGITAL_NUMBERS = "0123456789"; + +static const std::map> SUMMARY_OPTION_HEADER_STR_MAP = { + {DebuggerSummaryOption::MAX, {STATS_HEADER_MAX, STATS_HEADER_MAX}}, + {DebuggerSummaryOption::MIN, {STATS_HEADER_MIN, STATS_HEADER_MIN}}, + {DebuggerSummaryOption::MEAN, {STATS_HEADER_AVG, STATS_HEADER_AVG}}, + {DebuggerSummaryOption::L2NORM, {STATS_HEADER_L2NORM, STATS_CSV_HEADER_L2NORM}}, + {DebuggerSummaryOption::NAN_CNT, {STATS_HEADER_NAN, STATS_CSV_HEADER_NAN}}, + {DebuggerSummaryOption::NEG_INF_CNT, {STATS_HEADER_NEG_INF, STATS_HEADER_NEG_INF}}, + {DebuggerSummaryOption::POS_INF_CNT, {STATS_HEADER_POS_INF, STATS_HEADER_POS_INF}}, + {DebuggerSummaryOption::MD5, {STATS_HEADER_MD5, STATS_HEADER_MD5}}, }; const static std::map kDtypeTransMap = { @@ -91,7 +91,7 @@ public: std::string GetCsvHeader() const; std::string GetCsvValue() const; std::string GetPath() const {return path;} - bool empty() const {return stats.empty();}; + bool Empty() const {return stats.empty();}; static AclTensorStats CalTensorSummary(const AclTensorInfo& tensor, const std::vector& opt); static AclTensorStats ParseTensorSummary(const std::string& dumpPath, const std::string& input); @@ -114,13 +114,13 @@ private: void ParseInfoFromDumpPath(const std::string& dumpPath); std::string& operator[](DebuggerSummaryOption opt) { return stats[opt]; } - static constexpr const size_t bufferLen = 1024; + static constexpr const size_t BUFFER_LEN = 1024; }; void AclTensorStats::ParseInfoFromDumpPath(const std::string& dumpPath) { std::string filename; - if (FileUtils::GetFileSuffix(filename) == "csv") { + if (FileUtils::GetFileSuffix(dumpPath) == "csv") { filename = FileUtils::GetFileBaseName(dumpPath); } else { filename = FileUtils::GetFileName(dumpPath); @@ -159,7 +159,8 @@ AclTensorStats::AclTensorStats(const AclTensorInfo& tensor, const std::map& opt) +AclTensorStats AclTensorStats::CalTensorSummary(const AclTensorInfo& tensor, + const std::vector& opt) { DEBUG_FUNC_TRACE(); std::map summary; @@ -174,9 +175,9 @@ AclTensorStats AclTensorStats::CalTensorSummary(const AclTensorInfo& tensor, con static std::map ParseTensorSummaryHeaderOrder(const std::vector& segs) { std::map ret; - for (uint32_t pos = 0; pos < segs.size(); ++pos) { + for (size_t pos = 0; pos < segs.size(); ++pos) { const std::string& opt = segs[pos]; - for (auto it = summaryOptionHeaderStrMap.begin(); it != summaryOptionHeaderStrMap.end(); ++it) { + for (auto it = SUMMARY_OPTION_HEADER_STR_MAP.begin(); it != SUMMARY_OPTION_HEADER_STR_MAP.end(); ++it) { if (opt == it->second.first) { ret[pos] = it->first; break; @@ -188,14 +189,14 @@ static std::map ParseTensorSummaryHeaderOrder(c AclTensorStats AclTensorStats::ParseTensorSummary(const std::string& dumpPath, const std::string& input) { - constexpr const uint32_t optPosBase = 7; + constexpr const size_t optPosBase = 7; static std::map order; static uint32_t headerLen = 0; std::vector segs = FileUtils::SplitPath(input, ','); /* device计算统计量场景，各个kernel的统计项的顺序是相同的，只要计算一次即可 */ if (order.empty()) { - if (segs.size() <= optPosBase || segs[0] != kStatsHeaderInout) { + if (segs.size() <= optPosBase || segs[0] != STATS_HEADER_INOUT) { LOG_WARNING(DebuggerErrno::ERROR_INVALID_FORMAT, "Summary data miss header, some data may lose."); return AclTensorStats(); } @@ -211,7 +212,7 @@ AclTensorStats AclTensorStats::ParseTensorSummary(const std::string& dumpPath, c } /* 不重复解析header行 */ - if (segs[0] == kStatsHeaderInout) { + if (segs[0] == STATS_HEADER_INOUT) { return AclTensorStats(); } @@ -236,11 +237,11 @@ std::string AclTensorStats::GetCsvHeader() const return std::string(); } std::string ret; - ret.reserve(bufferLen); + ret.reserve(BUFFER_LEN); ret.append("Op Type,Op Name,Task ID,Stream ID,Timestamp,Input/Output,Slot,Data Size,Data Type,Format,Shape"); for (auto it = stats.begin(); it != stats.end(); it++) { ret.append(","); - ret.append(summaryOptionHeaderStrMap.at(it->first).second); + ret.append(SUMMARY_OPTION_HEADER_STR_MAP.at(it->first).second); } ret.append("\n"); @@ -254,7 +255,7 @@ std::string AclTensorStats::GetCsvValue() const } std::string ret; - ret.reserve(bufferLen); + ret.reserve(BUFFER_LEN); ret.append(opType).append(",").append(opName).append(",").append(taskID).append(",").append(streamID).append(",") \ .append(timestamp).append(",").append(inout).append(",").append(slot).append(",") .append(dataSize) \ .append(",").append(dataType).append(",").append(format).append(",").append(shape); @@ -282,7 +283,7 @@ std::string AclDumpDataProcessor::ToString() const std::to_string(totalLen) + ")"; } -DebuggerErrno AclDumpDataProcessor::PushData(const acldumpChunk *chunk) +DebuggerErrno AclDumpDataProcessor::PushData(const AclDumpChunk *chunk) { DEBUG_FUNC_TRACE(); if (completed) { @@ -297,8 +298,15 @@ DebuggerErrno AclDumpDataProcessor::PushData(const acldumpChunk *chunk) } size_t len = chunk->bufLen; + if (len == 0) { + LOG_ERROR(DebuggerErrno::ERROR_INVALID_VALUE, ToString() + ": invalid value(cached size " + + std::to_string(totalLen) + ", receiving size " + std::to_string(len) + ")."); + errorOccurred = true; + return DebuggerErrno::ERROR_INVALID_VALUE; + } + /* 防止正负翻转 */ - if (SIZE_MAX - len < totalLen || totalLen + len > kMaxDataLen || len == 0) { + if (SIZE_MAX - len < totalLen || totalLen + len > MAX_DATA_LEN) { LOG_ERROR(DebuggerErrno::ERROR_BUFFER_OVERFLOW, ToString() + ": buffer overflow(cached size " + std::to_string(totalLen) + ", receiving size " + std::to_string(len) + ")."); errorOccurred = true; @@ -313,7 +321,10 @@ DebuggerErrno AclDumpDataProcessor::PushData(const acldumpChunk *chunk) return DebuggerErrno::ERROR_NO_MEMORY; } - if (memcpy(p->data(), chunk->dataBuf, len) == nullptr) { + /* vector p根据chunk->dataBuf的长度，即len，申请创建，所以无需校验空间大小 */ + try { + std::copy(chunk->dataBuf, chunk->dataBuf + len, p->begin()); + } catch (const std::exception& e) { LOG_ERROR(DebuggerErrno::ERROR_SYSCALL_FAILED, ToString() + ": Failed to copy data;"); delete p; errorOccurred = true; @@ -361,9 +372,11 @@ DebuggerErrno AclDumpDataProcessor::ConcatenateData() } size_t offset = 0; - uint8_t* msg = p->data(); while (!buffer.empty()) { - if (memcpy(msg + offset, buffer.front()->data(), buffer.front()->size()) == nullptr) { + /* vector p根据buffer里所有vector的总长度，即totalLen，申请创建，所以无需校验空间大小 */ + try { + std::copy(buffer.front()->begin(), buffer.front()->end(), p->begin() + offset); + } catch (const std::exception& e) { delete p; LOG_ERROR(DebuggerErrno::ERROR_SYSCALL_FAILED, "Data processor(" + dumpPath + "): Failed to copy."); return DebuggerErrno::ERROR_SYSCALL_FAILED; @@ -405,17 +418,17 @@ static nlohmann::json ParseOverflowInfo(const uint8_t* data) DEBUG_FUNC_TRACE(); uint32_t index = 0; nlohmann::json overflowInfo; - uint64_t modelId = DataUtils::UnpackUint64Value_Le(data); - index += kUint64Size; - uint64_t streamId = DataUtils::UnpackUint64Value_Le(data + index); - index += kUint64Size; - uint64_t taskId = DataUtils::UnpackUint64Value_Le(data + index); - index += kUint64Size; - uint64_t taskType = DataUtils::UnpackUint64Value_Le(data + index); - index += kUint64Size; - uint64_t pcStart = DataUtils::UnpackUint64Value_Le(data + index); - index += kUint64Size; - uint64_t paraBase = DataUtils::UnpackUint64Value_Le(data + index); + uint64_t modelId = DataUtils::UnpackUint64ValueLe(data); + index += UINT64_SIZE; + uint64_t streamId = DataUtils::UnpackUint64ValueLe(data + index); + index += UINT64_SIZE; + uint64_t taskId = DataUtils::UnpackUint64ValueLe(data + index); + index += UINT64_SIZE; + uint64_t taskType = DataUtils::UnpackUint64ValueLe(data + index); + index += UINT64_SIZE; + uint64_t pcStart = DataUtils::UnpackUint64ValueLe(data + index); + index += UINT64_SIZE; + uint64_t paraBase = DataUtils::UnpackUint64ValueLe(data + index); overflowInfo["model_id"] = modelId; overflowInfo["stream_id"] = streamId; @@ -431,30 +444,30 @@ static DebuggerErrno DumpOpDebugDataToDisk(const std::string& dumpPath, AclDumpM { DEBUG_FUNC_TRACE(); std::string outPath = dumpPath + ".output."; - uint32_t num = dumpData.output().size(); + uint32_t num = static_cast(dumpData.output().size()); for (uint32_t slot = 0; slot < num; slot++) { uint32_t offset = 0; // parse DHA Atomic Add info nlohmann::json dhaAtomicAddInfo = ParseOverflowInfo(data + offset); - offset += kDhaAtomicAddInfoSize; + offset += DHA_ATOMIC_ADD_INFO_SIZE; // parse L2 Atomic Add info nlohmann::json l2AtomicAddInfo = ParseOverflowInfo(data + offset); - offset += kL2AtomicAddInfoSize; + offset += L2_ATOMIC_ADD_INFO_SIZE; // parse AICore info nlohmann::json aiCoreInfo = ParseOverflowInfo(data + offset); - offset += kAiCoreInfoSize; + offset += AICORE_INFO_SIZE; // parse DHA Atomic Add status - dhaAtomicAddInfo["status"] = DataUtils::UnpackUint64Value_Le(data + offset); - offset += kDhaAtomicAddStatusSize; + dhaAtomicAddInfo["status"] = DataUtils::UnpackUint64ValueLe(data + offset); + offset += DHA_ATOMIC_ADD_STATUS_SIZE; // parse L2 Atomic Add status - l2AtomicAddInfo["status"] = DataUtils::UnpackUint64Value_Le(data + offset); - offset += kL2AtomicAddStatusSize; + l2AtomicAddInfo["status"] = DataUtils::UnpackUint64ValueLe(data + offset); + offset += L2_ATOMIC_ADD_STATUS_SIZE; // parse AICore status - uint64_t kernelCode = DataUtils::UnpackUint64Value_Le(data + offset); - offset += kUint64Size; - uint64_t blockIdx = DataUtils::UnpackUint64Value_Le(data + offset); - offset += kUint64Size; - uint64_t status = DataUtils::UnpackUint64Value_Le(data + offset); + uint64_t kernelCode = DataUtils::UnpackUint64ValueLe(data + offset); + offset += UINT64_SIZE; + uint64_t blockIdx = DataUtils::UnpackUint64ValueLe(data + offset); + offset += UINT64_SIZE; + uint64_t status = DataUtils::UnpackUint64ValueLe(data + offset); aiCoreInfo["kernel_code"] = DataUtils::U64ToHexString(kernelCode); aiCoreInfo["block_idx"] = blockIdx; aiCoreInfo["status"] = status; @@ -530,8 +543,11 @@ static std::string MappingFilePath(const std::string& originPath) return std::string(); } - DebuggerErrno ret; - FileUtils::CreateDir(dir); + DebuggerErrno ret = FileUtils::CreateDir(dir); + if (ret != DebuggerErrno::OK) { + LOG_ERROR(DebuggerErrno::ERROR, "Failed to create directory " + dir + "."); + return std::string(); + } std::ofstream ofs; constexpr const char* mapFileName = "mapping.csv"; @@ -570,7 +586,8 @@ static DebuggerErrno StandardizedDumpPath(std::string& originPath) return DebuggerErrno::OK; } -static std::string GenDataPath(const std::string& path) { +static std::string GenDataPath(const std::string& path) +{ LOG_DEBUG("Original acl data path is " + path); std::string outputPath = DebuggerConfig::GetInstance().GetOutputPath(); std::string dataPath; @@ -592,7 +609,8 @@ static std::string GenDataPath(const std::string& path) { } /* * ACL 接口返回数据的路径格式如下 - * {dump_path}/rank_{rank_id}/{time stamp}/step_{step_id}/{time}/{device_id}/{model_name}/{model_id}/{iteration_id}/{data name} + * {dump_path}/rank_{rank_id}/{time stamp}/step_{step_id}/{time} + /{device_id}/{model_name}/{model_id}/{iteration_id}/{data name} * items[0] 表示 rank_{rank_id} * items[1] 表示 {time stamp} * items[2] 表示 step_{step_id} @@ -652,15 +670,15 @@ static DebuggerErrno DumpOneAclTensorFmtNpy(AclTensorInfo& tensor) AclDtype dstDtype = it->second; ret = AclTensor::TransDtype(tensor, dstDtype); if (ret != DebuggerErrno::OK) { - LOG_ERROR(ret, tensor + ": Failed to transform dtype from " + DataUtils::GetDTypeString(it->first) + " to " + - DataUtils::GetDTypeString(it->second)+ "."); + LOG_ERROR(ret, tensor + ": Failed to transform dtype from " + + DataUtils::GetDTypeString(it->first) + " to " + + DataUtils::GetDTypeString(it->second)+ "."); return ret; } } // dump_path: dump_dir/op_type.op_name.task_id.stream_id.timestamp std::string dumpPathSlot = tensor.dumpPath + GetTensorInfoSuffix(tensor) + "." + NPY_SUFFIX; - if (StandardizedDumpPath(dumpPathSlot) != DebuggerErrno::OK) { LOG_ERROR(DebuggerErrno::ERROR, "Failed to standardize path " + dumpPathSlot + "."); return DebuggerErrno::ERROR; @@ -686,7 +704,7 @@ static DebuggerErrno DumpOneAclTensorFmtNpy(AclTensorInfo& tensor) static DebuggerErrno WriteOneTensorStatToDisk(const AclTensorStats& stat) { DEBUG_FUNC_TRACE(); - if (stat.empty()) { + if (stat.Empty()) { return DebuggerErrno::OK; } @@ -694,7 +712,7 @@ static DebuggerErrno WriteOneTensorStatToDisk(const AclTensorStats& stat) /* 此处防止多进程间竞争，使用文件锁，故使用C风格接口 */ uint32_t retry = 100; uint32_t interval = 10; - if (FileUtils::IsPathExist(dumpfile) && !FileUtils::IsRegularFile(dumpfile)) { + if (FileUtils::CheckFileBeforeCreateOrWrite(dumpfile, true) != DebuggerErrno::OK) { LOG_ERROR(DebuggerErrno::ERROR_FILE_ALREADY_EXISTS, "File " + dumpfile + " exists and has invalid format."); return DebuggerErrno::ERROR_FILE_ALREADY_EXISTS; } @@ -887,7 +905,7 @@ DebuggerErrno AclDumpDataProcessor::DumpToDisk() const std::string dataPath = GenDataPath(dumpPath); DebuggerErrno ret; - if (FileUtils::GetFileName(dumpPath).find(debugFileSign) == 0 && + if (FileUtils::GetFileName(dumpPath).find(DEBUG_FILE_SIGN) == 0 && DebuggerConfig::GetInstance().GetOverflowCheckCfg() != nullptr) { ret = DumpOpDebugDataToDisk(dataPath, dumpData, msg + dataSegOffset, dataSegLen); } else if (DebuggerConfig::GetInstance().GetStatisticsCfg() != nullptr && diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.hpp b/debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.h similarity index 82% rename from debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.hpp rename to debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.h index 4ce2ab6e8c8709437791aba9699ec76184cb6761..227f0f45dc3fb621cb687c5199f14576d9a1699e 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/AclDumpDataProcessor.h @@ -20,23 +20,23 @@ #include #include -#include "include/ErrorCode.hpp" -#include "base/DebuggerConfig.hpp" -#include "third_party/ACL/AclApi.hpp" +#include "include/ErrorCode.h" +#include "base/DebuggerConfig.h" +#include "third_party/ACL/AclApi.h" namespace MindStudioDebugger { -constexpr size_t kMaxDataLen = 4ULL * 1024 * 1024 * 1024; +constexpr size_t MAX_DATA_LEN = 4ULL * 1024 * 1024 * 1024; class AclDumpDataProcessor { public: - AclDumpDataProcessor(const std::string& path, const std::vector& opts) : - dumpPath{path}, hostAnalysisOpts{opts} {}; + AclDumpDataProcessor(const std::string& path, const std::vector& opts) + : dumpPath{path}, hostAnalysisOpts{opts} {}; ~AclDumpDataProcessor(); bool IsCompleted() const {return completed;} bool ErrorOccurred() const {return errorOccurred;} - DebuggerErrno PushData(const acldumpChunk *chunk); + DebuggerErrno PushData(const AclDumpChunk *chunk); DebuggerErrno DumpToDisk(); std::string ToString() const; diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.cpp b/debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.cpp index 805a6a7a0a24bb1fee1472698511d53beb7a35a6..7c103e42b8a8177f95c4936b94ff1192bb5cf696 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.cpp @@ -19,51 +19,51 @@ #include #include -#include "include/Macro.hpp" -#include "utils/FileUtils.hpp" -#include "utils/FileOperation.hpp" -#include "third_party/ACL/AclApi.hpp" -#include "base/Environment.hpp" -#include "base/ErrorInfos.hpp" -#include "AclDumper.hpp" +#include "include/Macro.h" +#include "utils/FileUtils.h" +#include "utils/FileOperation.h" +#include "third_party/ACL/AclApi.h" +#include "base/Environment.h" +#include "base/ErrorInfosManager.h" +#include "AclDumper.h" namespace MindStudioDebugger { -constexpr const char* kAclDumpScene = "dump_scene"; -constexpr const char* kSceneNormal = "normal"; -constexpr const char* kSceneException ="lite_exception"; +constexpr const char* ACL_DUMP_SCENE = "dump_scene"; +constexpr const char* SCENE_NORMAL = "normal"; +constexpr const char* SCENE_EXCEPTION = "lite_exception"; -constexpr const char* kAclDumpPath = "dump_path"; -constexpr const char* kAclDumpStep = "dump_step"; +constexpr const char* ACL_DUMP_PATH = "dump_path"; +constexpr const char* ACL_DUMP_STEP = "dump_step"; -constexpr const char* kAclDumpList = "dump_list"; -constexpr const char* kAclDumpLayer = "layer"; -constexpr const char* kAclDumpModel = "model_name"; +constexpr const char* ACL_DUMP_LIST = "dump_list"; +constexpr const char* ACL_DUMP_LAYER = "layer"; +constexpr const char* ACL_DUMP_MODEL_NAME = "model_name"; -constexpr const char* kAclDumpMode = "dump_mode"; -constexpr const char* kAclModeInput = "input"; -constexpr const char* kAclModeOutput = "output"; -constexpr const char* kAclModeAll = "all"; +constexpr const char* ACL_DUMP_MODE = "dump_mode"; +constexpr const char* ACL_MODE_INPUT = "input"; +constexpr const char* ACL_MODE_OUTPUT = "output"; +constexpr const char* ACL_MODE_ALL = "all"; -constexpr const char* kAclDumpOpSwitch = "dump_op_switch"; -constexpr const char* kAclDumpDebug = "dump_debug"; -constexpr const char* kAclSwitchOn = "on"; -constexpr const char* kAclSwitchOff = "off"; +constexpr const char* DUMP_OP_SWITCH = "dump_op_switch"; +constexpr const char* ACL_DUMP_DEBUG = "dump_debug"; +constexpr const char* ACL_SWITCH_ON = "on"; +constexpr const char* ACL_SWITCH_OFF = "off"; -constexpr const char* kAclDumpData = "dump_data"; -constexpr const char* kAclDumpTensor = "tensor"; -constexpr const char* kAclDumpStats = "stats"; +constexpr const char* ACL_DUMP_DATA = "dump_data"; +constexpr const char* ACL_DUMP_TENSOR = "tensor"; +constexpr const char* ACL_DUMP_STATS = "stats"; -constexpr const char* kAclDumpStatsOpt = "dump_stats"; -constexpr const char* kAclDumpStatsMax = "Max"; -constexpr const char* kAclDumpStatsMin = "Min"; -constexpr const char* kAclDumpStatsAvg = "Avg"; -constexpr const char* kAclDumpStatsNorn = "L2norm"; -constexpr const char* kAclDumpStatsNan = "Nan"; -constexpr const char* kAclDumpStatsNegInf = "Negative Inf"; -constexpr const char* kAclDumpStatsPosInf = "Positive Inf"; +constexpr const char* ACL_DUMP_STATS_OPT = "dump_stats"; +constexpr const char* ACL_DUMP_STATS_MAX = "Max"; +constexpr const char* ACL_DUMP_STATS_MIN = "Min"; +constexpr const char* ACL_DUMP_STATS_AVG = "Avg"; +constexpr const char* ACL_DUMP_STATS_NORM = "L2norm"; +constexpr const char* ACL_DUMP_STATS_NAN = "Nan"; +constexpr const char* ACL_DUMP_STATS_NEG_INF = "Negative Inf"; +constexpr const char* ACL_DUMP_STATS_POS_INF = "Positive Inf"; -constexpr const size_t kProcessorNumMax = 100; +constexpr const size_t PROCESSOR_NUM_MAX = 100; inline std::string GenAclJsonPath(const std::string& dumpPath, uint32_t rank) { @@ -74,14 +74,14 @@ inline std::string GenAclJsonPath(const std::string& dumpPath, uint32_t rank) static std::string GenDumpInoutString(DebuggerDataInOut mode) { static std::map dumpModeMap = { - {DebuggerDataInOut::INOUT_INPUT, kAclModeInput}, - {DebuggerDataInOut::INOUT_OUTPUT, kAclModeOutput}, - {DebuggerDataInOut::INOUT_BOTH, kAclModeAll}, + {DebuggerDataInOut::INOUT_INPUT, ACL_MODE_INPUT}, + {DebuggerDataInOut::INOUT_OUTPUT, ACL_MODE_OUTPUT}, + {DebuggerDataInOut::INOUT_BOTH, ACL_MODE_ALL}, }; auto it = dumpModeMap.find(mode); if (it == dumpModeMap.end()) { - return kAclModeAll; + return ACL_MODE_ALL; } else { return it->second; } @@ -90,13 +90,13 @@ static std::string GenDumpInoutString(DebuggerDataInOut mode) static std::vector GenStatsOptions(const std::vector& options) { static std::map summaryOptMap = { - {DebuggerSummaryOption::MAX, kAclDumpStatsMax}, - {DebuggerSummaryOption::MIN, kAclDumpStatsMin}, - {DebuggerSummaryOption::MEAN, kAclDumpStatsAvg}, - {DebuggerSummaryOption::L2NORM, kAclDumpStatsNorn}, - {DebuggerSummaryOption::NAN_CNT, kAclDumpStatsNan}, - {DebuggerSummaryOption::NEG_INF_CNT, kAclDumpStatsNegInf}, - {DebuggerSummaryOption::POS_INF_CNT, kAclDumpStatsPosInf}, + {DebuggerSummaryOption::MAX, ACL_DUMP_STATS_MAX}, + {DebuggerSummaryOption::MIN, ACL_DUMP_STATS_MIN}, + {DebuggerSummaryOption::MEAN, ACL_DUMP_STATS_AVG}, + {DebuggerSummaryOption::L2NORM, ACL_DUMP_STATS_NORM}, + {DebuggerSummaryOption::NAN_CNT, ACL_DUMP_STATS_NAN}, + {DebuggerSummaryOption::NEG_INF_CNT, ACL_DUMP_STATS_NEG_INF}, + {DebuggerSummaryOption::POS_INF_CNT, ACL_DUMP_STATS_POS_INF}, }; std::vector output; @@ -156,7 +156,7 @@ bool AclDumper::IsOverflowCompleted() return overflowNums != -1 && realOverflowNums > overflowNums; } -void AclDumper::CountOverflowNumbers(const acldumpChunk* chunk) +void AclDumper::CountOverflowNumbers(const AclDumpChunk* chunk) { if (IsOverflowCompleted() || !isOverflowDump || !chunk->isLastChunk) { return; @@ -194,19 +194,19 @@ DebuggerErrno AclDumper::AclDumpGenTensorJson(std::shared_ptrinout); - aclDumpJson[kAclDumpData] = kAclDumpTensor; - aclDumpJson[kAclDumpList] = nlohmann::json::array(); - aclDumpJson[kAclDumpOpSwitch] = kAclSwitchOn; + aclDumpJson[ACL_DUMP_PATH] = fullDumpPath; + aclDumpJson[ACL_DUMP_MODE] = GenDumpInoutString(dumpTensorCfg->inout); + aclDumpJson[ACL_DUMP_DATA] = ACL_DUMP_TENSOR; + aclDumpJson[ACL_DUMP_LIST] = nlohmann::json::array(); + aclDumpJson[DUMP_OP_SWITCH] = ACL_SWITCH_ON; if (!needDump) { /* 这里沿用mindspore框架的方案，用一个大数0x7FFFFFFF表示不需要dump；这个方案非常奇怪，后续可以看下能否优化 */ - aclDumpJson[kAclDumpStep] = std::to_string(INT_MAX); + aclDumpJson[ACL_DUMP_STEP] = std::to_string(INT_MAX); } else { std::vector kernelsList = dumpTensorCfg->matcher.GenRealKernelList(kernels); if (!kernelsList.empty()) { - aclDumpJson[kAclDumpList].push_back({{kAclDumpLayer, kernelsList}}); + aclDumpJson[ACL_DUMP_LIST].push_back({{ACL_DUMP_LAYER, kernelsList}}); } } @@ -230,25 +230,26 @@ DebuggerErrno AclDumper::AclDumpGenStatJson(std::shared_ptr fullDumpPath = dumpPath; } - aclDumpJson[kAclDumpPath] = fullDumpPath; - aclDumpJson[kAclDumpMode] = GenDumpInoutString(statisticsCfg->inout); - aclDumpJson[kAclDumpList] = nlohmann::json::array(); - aclDumpJson[kAclDumpOpSwitch] = kAclSwitchOn; + aclDumpJson[ACL_DUMP_PATH] = fullDumpPath; + aclDumpJson[ACL_DUMP_MODE] = GenDumpInoutString(statisticsCfg->inout); + aclDumpJson[ACL_DUMP_LIST] = nlohmann::json::array(); + aclDumpJson[DUMP_OP_SWITCH] = ACL_SWITCH_ON; /* 如果需要host侧分析，下给acl的任务还是dump tensor，然后在host侧转成统计量 */ if (!hostAnalysisOpt.empty()) { - aclDumpJson[kAclDumpData] = kAclDumpTensor; + aclDumpJson[ACL_DUMP_DATA] = ACL_DUMP_TENSOR; } else { - aclDumpJson[kAclDumpData] = kAclDumpStats; - aclDumpJson[kAclDumpStatsOpt] = GenStatsOptions(statisticsCfg->summaryOption); + aclDumpJson[ACL_DUMP_DATA] = ACL_DUMP_STATS; + aclDumpJson[ACL_DUMP_STATS_OPT] = GenStatsOptions(statisticsCfg->summaryOption); } if (!needDump) { - aclDumpJson[kAclDumpStep] = std::to_string(INT_MAX); + aclDumpJson[ACL_DUMP_STEP] = std::to_string(INT_MAX); } else { std::vector kernelsList = statisticsCfg->matcher.GenRealKernelList(kernels); - if (!kernelsList.empty()){ - aclDumpJson[kAclDumpList].push_back({{kAclDumpLayer, kernelsList}}); + if (!kernelsList.empty()) + { + aclDumpJson[ACL_DUMP_LIST].push_back({{ACL_DUMP_LAYER, kernelsList}}); } } @@ -277,10 +278,10 @@ DebuggerErrno AclDumper::AclDumpGenOverflowJson(std::shared_ptrfileName); auto it = dataProcessors.find(dumpPath); if (it == dataProcessors.end()) { - if (dataProcessors.size() > kProcessorNumMax) { + if (dataProcessors.size() > PROCESSOR_NUM_MAX) { LOG_ERROR(DebuggerErrno::ERROR_BUFFER_OVERFLOW, "The number of processors has reached the upper limit."); return; } @@ -429,7 +430,7 @@ void AclDumper::SetDump(uint32_t rank, uint32_t curStep, ExtArgs& args) if (!initialized) { ret = Initialize(); - if(ret != DebuggerErrno::OK) { + if (ret != DebuggerErrno::OK) { LOG_ERROR(ret, "AclDumper initialization failed."); return; } @@ -458,8 +459,7 @@ void AclDumper::SetDump(uint32_t rank, uint32_t curStep, ExtArgs& args) return; } - aclError aclRet; - aclRet = CALL_ACL_API(aclmdlInitDump); + aclError aclRet = CALL_ACL_API(AclmdlInitDump); if (aclRet != ACL_SUCCESS) { LOG_ERROR(DebuggerErrno::ERROR_EXTERNAL_API_ERROR, "Failed to init acldump(" + std::to_string(aclRet) + ")."); @@ -467,7 +467,7 @@ void AclDumper::SetDump(uint32_t rank, uint32_t curStep, ExtArgs& args) } const std::string& dumpPath = DebuggerConfig::GetInstance().GetOutputPath(); - aclRet = CALL_ACL_API(aclmdlSetDump, GenAclJsonPath(dumpPath, rank).c_str()); + aclRet = CALL_ACL_API(AclmdlSetDump, GenAclJsonPath(dumpPath, rank).c_str()); if (aclRet != ACL_SUCCESS) { LOG_ERROR(DebuggerErrno::ERROR_EXTERNAL_API_ERROR, "Failed to enable acldump(" + std::to_string(aclRet) + ")."); @@ -485,51 +485,53 @@ void AclDumper::FinalizeDump(ExtArgs& args) return; } - CALL_ACL_API(aclrtSynchronizeDevice); - aclError aclRet = CALL_ACL_API(aclmdlFinalizeDump); + CALL_ACL_API(AclrtSynchronizeDevice); + aclError aclRet = CALL_ACL_API(AclmdlFinalizeDump); if (aclRet != ACL_SUCCESS) { LOG_ERROR(DebuggerErrno::ERROR_EXTERNAL_API_ERROR, "Failed to finalize acldump(" + std::to_string(aclRet) + ")."); - } aclDumpHasSet = false; } -void KernelInitDump() { - if (AscendCLApi::LoadAclApi() != DebuggerErrno::OK) { - return; - } +void KernelInitDump() +{ + if (AscendCLApi::LoadAclApi() != DebuggerErrno::OK) { + return; + } - DebuggerErrno ret = InitAcl(); - if (ret != DebuggerErrno::OK) { - LOG_ERROR(ret, "Failed to call InitAcl."); - return; - } - auto aclRet = CALL_ACL_API(aclmdlInitDump); - if (aclRet != ACL_SUCCESS) { + DebuggerErrno ret = InitAcl(); + if (ret != DebuggerErrno::OK) { + LOG_ERROR(ret, "Failed to call InitAcl."); + return; + } + auto aclRet = CALL_ACL_API(AclmdlInitDump); + if (aclRet != ACL_SUCCESS) { LOG_ERROR(DebuggerErrno::ERROR_EXTERNAL_API_ERROR, "Failed to init acldump(" + std::to_string(aclRet) + ")."); return; - } + } } -void KernelSetDump(const std::string &filePath) { - std::string dumpPath = FileUtils::GetAbsPath(filePath); - auto aclRet = CALL_ACL_API(aclmdlSetDump, dumpPath.c_str()); - if (aclRet != ACL_SUCCESS) { +void KernelSetDump(const std::string &filePath) +{ + std::string dumpPath = FileUtils::GetAbsPath(filePath); + auto aclRet = CALL_ACL_API(AclmdlSetDump, dumpPath.c_str()); + if (aclRet != ACL_SUCCESS) { LOG_ERROR(DebuggerErrno::ERROR_EXTERNAL_API_ERROR, "Failed to enable acldump(" + std::to_string(aclRet) + ")."); return; - } + } } -void KernelFinalizeDump() { - CALL_ACL_API(aclrtSynchronizeDevice); - auto aclRet = CALL_ACL_API(aclmdlFinalizeDump); - if (aclRet != ACL_SUCCESS) { +void KernelFinalizeDump() +{ + CALL_ACL_API(AclrtSynchronizeDevice); + auto aclRet = CALL_ACL_API(AclmdlFinalizeDump); + if (aclRet != ACL_SUCCESS) { LOG_ERROR(DebuggerErrno::ERROR_EXTERNAL_API_ERROR, "Failed to finalize acldump(" + std::to_string(aclRet) + ")."); - } + } } } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.hpp b/debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.h similarity index 87% rename from debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.hpp rename to debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.h index 6985df65e166101c08501e5e206e003bda494b9a..b4316b18418acf343987f5443d230ea4039c1612 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/AclDumper.h @@ -21,17 +21,18 @@ #include #include -#include "include/ExtArgs.hpp" -#include "base/DebuggerConfig.hpp" -#include "AclDumpDataProcessor.hpp" +#include "include/ExtArgs.h" +#include "base/DebuggerConfig.h" +#include "AclDumpDataProcessor.h" namespace MindStudioDebugger { class AclDumper { public: - static AclDumper& GetInstance() { - static AclDumper instance_; - return instance_; + static AclDumper& GetInstance() + { + static AclDumper dumperInstance; + return dumperInstance; } static bool IsIterNeedDump(uint32_t iterId); @@ -39,7 +40,7 @@ public: void SetDump(uint32_t rank, uint32_t curStep, ExtArgs& args); void FinalizeDump(ExtArgs& args); - void OnAclDumpCallBack(const acldumpChunk* chunk, int32_t len); + void OnAclDumpCallBack(const AclDumpChunk* chunk, int32_t len); std::string GetDumpPath(uint32_t curStep) const; @@ -58,7 +59,7 @@ private: uint32_t curStep, const char** kernels); DebuggerErrno AclDumpGenOverflowJson(std::shared_ptr overflowCfg, uint32_t rank, uint32_t curStep); - void CountOverflowNumbers(const acldumpChunk* chunk); + void CountOverflowNumbers(const AclDumpChunk* chunk); bool IsOverflowCompleted(); bool initialized{false}; diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.cpp b/debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.cpp index 4a5ec4c555198015603d7cc1446be66fda05765d..2ff83ee8d8b5bbcc9f7cd486dfa41b2e1c756480 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.cpp @@ -22,10 +22,10 @@ #include #include -#include "utils/DataUtils.hpp" -#include "utils/MathUtils.hpp" -#include "base/ErrorInfos.hpp" -#include "AclTensor.hpp" +#include "utils/DataUtils.h" +#include "utils/MathUtils.h" +#include "base/ErrorInfosManager.h" +#include "AclTensor.h" namespace MindStudioDebugger { namespace AclDumpMsg = toolkit::dumpdata; @@ -33,21 +33,21 @@ namespace AclTensor { using namespace MathUtils; -constexpr int64_t kCubeSize = 16; -constexpr int64_t kCube16 = kCubeSize; -constexpr int64_t kCube32 = 32; -constexpr int64_t kCube64 = 64; -constexpr int64_t kCubeSize_C04 = 4; - -constexpr size_t hwH = 1; -constexpr size_t hwW = 2; -constexpr size_t fnzW1 = 4; -constexpr size_t fnzH1 = 3; -constexpr size_t fnzH0 = 2; -constexpr size_t fnzW0 = 1; -constexpr size_t fzN0 = 1; -constexpr size_t fzNi = 2; -constexpr size_t fzC0 = 3; +constexpr int64_t CUBE_SIZE = 16; +constexpr int64_t CUBE_16 = CUBE_SIZE; +constexpr int64_t CUBE_32 = 32; +constexpr int64_t CUBE_64 = 64; +constexpr int64_t CUBE_SIZE_C04 = 4; + +constexpr size_t HW_H = 1; +constexpr size_t HW_W = 2; +constexpr size_t FNZ_W1 = 4; +constexpr size_t FNZ_H1 = 3; +constexpr size_t FNZ_H0 = 2; +constexpr size_t FNZ_W0 = 1; +constexpr size_t FZ_N0 = 1; +constexpr size_t FZ_NI = 2; +constexpr size_t FZ_C0 = 3; using TensorTransFunc = DebuggerErrno (*)(AclTensorInfo &); @@ -94,21 +94,20 @@ const static std::unordered_set kSupportedFormat = { AclFormat::FORMAT_DHWNC, AclFormat::FORMAT_NDC1HWC0, AclFormat::FORMAT_FRACTAL_Z_3D, - AclFormat::FORMAT_C1HWNCoC0, + AclFormat::FORMAT_C1HWNCOC0, AclFormat::FORMAT_FRACTAL_NZ, AclFormat::FORMAT_FRACTAL_ZN_LSTM, AclFormat::FORMAT_NCL, }; const static std::map, TensorTransFunc> formatTransFuncMap = { - /* {{from, to}, function} */ {{AclFormat::FORMAT_HWCN, AclFormat::FORMAT_NCHW}, nullptr}, {{AclFormat::FORMAT_NHWC, AclFormat::FORMAT_NCHW}, nullptr}, {{AclFormat::FORMAT_FRACTAL_Z, AclFormat::FORMAT_NCHW}, FRAC_Z_TO_NCHW}, {{AclFormat::FORMAT_FRACTAL_NZ, AclFormat::FORMAT_NCHW}, FRAC_NZ_TO_NCHW}, {{AclFormat::FORMAT_NC1HWC0, AclFormat::FORMAT_NCHW}, NC1HWC0_TO_NCHW}, {{AclFormat::FORMAT_NDC1HWC0, AclFormat::FORMAT_NCHW}, NDC1HWC0_TO_NCDHW}, - {{AclFormat::FORMAT_C1HWNCoC0, AclFormat::FORMAT_NCHW}, C1HWNCoC0_TO_NCHW}, + {{AclFormat::FORMAT_C1HWNCOC0, AclFormat::FORMAT_NCHW}, C1HWNCoC0_TO_NCHW}, {{AclFormat::FORMAT_NC1HWC0_C04, AclFormat::FORMAT_NCHW}, NC1HWC0_C04_TO_NCHW}, {{AclFormat::FORMAT_FRACTAL_Z_3D, AclFormat::FORMAT_NCHW}, FRAC_Z3D_TO_NCDHW}, }; @@ -164,7 +163,8 @@ const static std::unordered_map formatTrans {AclDumpMsg::OutputFormat::FORMAT_NC1HWC0_C04, AclFormat::FORMAT_NC1HWC0_C04}, {AclDumpMsg::OutputFormat::FORMAT_FRACTAL_Z_C04, AclFormat::FORMAT_FRACTAL_Z_C04}, {AclDumpMsg::OutputFormat::FORMAT_CHWN, AclFormat::FORMAT_CHWN}, - {AclDumpMsg::OutputFormat::FORMAT_FRACTAL_DECONV_SP_STRIDE8_TRANS, AclFormat::FORMAT_FRACTAL_DECONV_SP_STRIDE8_TRANS}, + {AclDumpMsg::OutputFormat::FORMAT_FRACTAL_DECONV_SP_STRIDE8_TRANS, + AclFormat::FORMAT_FRACTAL_DECONV_SP_STRIDE8_TRANS}, {AclDumpMsg::OutputFormat::FORMAT_HWCN, AclFormat::FORMAT_HWCN}, {AclDumpMsg::OutputFormat::FORMAT_NC1KHKWHWC0, AclFormat::FORMAT_NC1KHKWHWC0}, {AclDumpMsg::OutputFormat::FORMAT_BN_WEIGHT, AclFormat::FORMAT_BN_WEIGHT}, @@ -174,7 +174,7 @@ const static std::unordered_map formatTrans {AclDumpMsg::OutputFormat::FORMAT_HASHTABLE_LOOKUP_VALUE, AclFormat::FORMAT_HASHTABLE_LOOKUP_VALUE}, {AclDumpMsg::OutputFormat::FORMAT_HASHTABLE_LOOKUP_OUTPUT, AclFormat::FORMAT_HASHTABLE_LOOKUP_OUTPUT}, {AclDumpMsg::OutputFormat::FORMAT_HASHTABLE_LOOKUP_HITS, AclFormat::FORMAT_HASHTABLE_LOOKUP_HITS}, - {AclDumpMsg::OutputFormat::FORMAT_C1HWNCoC0, AclFormat::FORMAT_C1HWNCoC0}, + {AclDumpMsg::OutputFormat::FORMAT_C1HWNCoC0, AclFormat::FORMAT_C1HWNCOC0}, {AclDumpMsg::OutputFormat::FORMAT_MD, AclFormat::FORMAT_MD}, {AclDumpMsg::OutputFormat::FORMAT_NDHWC, AclFormat::FORMAT_NDHWC}, {AclDumpMsg::OutputFormat::FORMAT_FRACTAL_ZZ, AclFormat::FORMAT_FRACTAL_ZZ}, @@ -201,20 +201,20 @@ const static std::unordered_map formatTrans {AclDumpMsg::OutputFormat::FORMAT_C1HWC0, AclFormat::FORMAT_C1HWC0}, }; -enum kAxis4D : int { kN = 0, kC, kH, kW, kNchwDims }; +enum Axis4D : int { AXIS_N = 0, AXIS_C, AXIS_H, AXIS_W, NCHW_DIMS }; enum Axis5D : int { - N_ncdhw = 0, - C_ncdhw, - D_ncdhw, - H_ncdhw, - W_ncdhw, - kNcdhw, - N_ndc1hwc0 = 0, - D_ndc1hwc0, - C1_ndc1hwc0, - H_ndc1hwc0, - W_ndc1hwc0, - C0_ndc1hwc0 + N_NCDHW, + C_NCDHW, + D_NCDHW, + H_NCDHW, + W_NCDHW, + NCDHW, + N_NDC1HWC0, + D_NDC1HWC0, + C1_NDC1HWC0, + H_NDC1HWC0, + W_NDC1HWC0, + C0_NDC1HWC0 }; static inline AclDtype transAclDtype2MS(AclDumpMsg::OutputDataType dt) @@ -235,7 +235,8 @@ static inline AclFormat transAclFormat2MS(AclDumpMsg::OutputFormat fmt) return AclFormat::FORMAT_MAX; } -static size_t EleNumOfTensor(const AclTensorInfo& tensor, bool host = true) { +static size_t EleNumOfTensor(const AclTensorInfo& tensor, bool host = true) +{ size_t num = 1; const AclShape& shape = host ? tensor.hostShape : tensor.deviceShape; for (auto dim : shape) { @@ -244,23 +245,26 @@ static size_t EleNumOfTensor(const AclTensorInfo& tensor, bool host = true) { return 0; } - if (SIZE_MAX / dim < num) { + if (SIZE_MAX / dim < static_cast(num)) { throw std::out_of_range(tensor + ": Count of element over size_t."); } num *= static_cast(dim); } - return num; + return num; } -static inline size_t SizeOfAclDType(const AclTensorInfo& tensor) { +static inline size_t SizeOfAclDType(const AclTensorInfo& tensor) +{ return DataUtils::SizeOfDType(tensor.dtype); } -static inline size_t SizeOfAclDType(const AclDtype& dtype) { +static inline size_t SizeOfAclDType(const AclDtype& dtype) +{ return DataUtils::SizeOfDType(dtype); } -size_t SizeOfTensor(const AclTensorInfo& tensor, bool host) { +size_t SizeOfTensor(const AclTensorInfo& tensor, bool host) +{ size_t num = EleNumOfTensor(tensor, host); size_t eleSize = SizeOfAclDType(tensor); if (num != 0 && SIZE_MAX / num < eleSize) { @@ -269,16 +273,17 @@ size_t SizeOfTensor(const AclTensorInfo& tensor, bool host) { return num * eleSize; } -static inline int64_t GetCubeSizeByType(const AclDtype& dtype) { +static inline int64_t GetCubeSizeByType(const AclDtype& dtype) +{ if (dtype == AclDtype::DT_UINT8 || dtype == AclDtype::DT_INT8) { - return kCube32; + return CUBE_32; } if (dtype == AclDtype::DT_INT4) { - return kCube64; + return CUBE_64; } - return kCube16; + return CUBE_16; } static inline void AssertDim(const AclShape& shape, size_t dim) @@ -291,11 +296,14 @@ static inline void AssertDim(const AclShape& shape, size_t dim) static inline void AssertConsis(const AclTensorInfo& tensor) { - size_t tensor_size = EleNumOfTensor(tensor, false) * SizeOfAclDType(tensor); + size_t tensorSize = EleNumOfTensor(tensor, false) * SizeOfAclDType(tensor); // Processing dtype whose size < 1 // The ele num of quantization type(qint4*2) in MindSpore must be even. - if (tensor.dtype == AclDtype::DT_INT4) tensor_size = EleNumOfTensor(tensor, false) / 2; - if (tensor_size != tensor.dataSize) { + int int4_size_factor = 2; + if (tensor.dtype == AclDtype::DT_INT4) { + tensorSize = EleNumOfTensor(tensor, false) / int4_size_factor; + } + if (tensorSize != tensor.dataSize) { throw std::runtime_error(tensor + ": The internal data of Tensor is inconsistent."); } } @@ -325,8 +333,8 @@ AclTensorInfo ParseAttrsFromDumpData(const std::string& dumpPath, const uint8_t* for (auto d : tensor.original_shape().dim()) { if (d > INT64_MAX) { LOG_WARNING(DebuggerErrno::ERROR_VALUE_OVERFLOW, - "The value(" + std::to_string(d) + ") exceeds the max value of int64_t, " + - "this maybe caused by the unfixed shape operaters."); + "The value(" + std::to_string(d) + ") exceeds the max value of int64_t, " + + "this maybe caused by the unfixed shape operaters."); hShape.clear(); break; } @@ -335,7 +343,7 @@ AclTensorInfo ParseAttrsFromDumpData(const std::string& dumpPath, const uint8_t* // convert format to host format. It can be either NCHW or ND (non 4-dimemsions). AclFormat hFmt; - if (hShape.size() == kDim4) { + if (hShape.size() == DIM_4) { hFmt = AclFormat::FORMAT_NCHW; } else if (hShape.empty()) { hFmt = dFmt; @@ -347,7 +355,8 @@ AclTensorInfo ParseAttrsFromDumpData(const std::string& dumpPath, const uint8_t* } int32_t subFormat = tensor.sub_format(); - return AclTensorInfo{dumpPath, data, dtype, dtype, dFmt, hFmt, dShape, hShape, dataSize, subFormat, io, slot, dumpOriginData}; + return AclTensorInfo{dumpPath, data, dtype, dtype, dFmt, hFmt, + dShape, hShape, dataSize, subFormat, io, slot, dumpOriginData}; } template AclTensorInfo ParseAttrsFromDumpData( @@ -364,14 +373,14 @@ static inline void AllocTensorTransBuf(AclTensorInfo& tensor) static DebuggerErrno FRAC_Z_TO_NCHW_WITH_GROUPS(AclTensorInfo& tensor) { - AssertDim(tensor.hostShape, kDim4); + AssertDim(tensor.hostShape, DIM_4); AssertConsis(tensor); AllocTensorTransBuf(tensor); - auto nDim = tensor.hostShape[kN]; - auto cDim = tensor.hostShape[kC]; - auto hDim = tensor.hostShape[kH]; - auto wDim = tensor.hostShape[kW]; + auto nDim = tensor.hostShape[AXIS_N]; + auto cDim = tensor.hostShape[AXIS_C]; + auto hDim = tensor.hostShape[AXIS_H]; + auto wDim = tensor.hostShape[AXIS_W]; auto groups = tensor.subFormat; auto cinOri = cDim; auto coutOri = nDim / groups; @@ -382,7 +391,7 @@ static DebuggerErrno FRAC_Z_TO_NCHW_WITH_GROUPS(AclTensorInfo& tensor) } auto cubeK = GetCubeSizeByType(tensor.dtype); - auto eMult = std::min(Lcm(Lcm(cinOri, cubeK) / cinOri, Lcm(coutOri, kCubeSize) / cinOri), + auto eMult = std::min(Lcm(Lcm(cinOri, cubeK) / cinOri, Lcm(coutOri, CUBE_SIZE) / cinOri), static_cast(groups)); if (eMult == 0) { LOG_WARNING(DebuggerErrno::ERROR_INVALID_VALUE, @@ -391,11 +400,12 @@ static DebuggerErrno FRAC_Z_TO_NCHW_WITH_GROUPS(AclTensorInfo& tensor) } auto cinOpt = AlignCeil(eMult * cinOri, cubeK); - auto coutOpt = AlignCeil(eMult * coutOri, kCubeSize); + auto coutOpt = AlignCeil(eMult * coutOri, CUBE_SIZE); auto c1Dim = cinOpt / cubeK; const uint8_t* src = tensor.aclData; - uint8_t* dst = tensor.transBuf.data(); - auto dtypeSize = SizeOfAclDType(tensor); + auto dst = tensor.transBuf.begin(); + int64_t dtypeSize = static_cast(SizeOfAclDType(tensor)); + int64_t dstSize = static_cast(tensor.transBuf.size()); for (int64_t g = 0; g < groups; ++g) { for (int64_t c = 0; c < cDim; ++c) { @@ -411,8 +421,13 @@ static DebuggerErrno FRAC_Z_TO_NCHW_WITH_GROUPS(AclTensorInfo& tensor) (dstCi / cubeK) * hDim * wDim * coutOpt * cubeK + h * wDim * coutOpt * cubeK + w * coutOpt * cubeK + dstCo * cubeK + temporary; int64_t hstIdx = srcCo * cDim * hDim * wDim + c * hDim * wDim + h * wDim + w; - /* 此处由偏移计算逻辑保障不会越界读写 */ - std::memcpy(dst + hstIdx * dtypeSize, src + devIdx * dtypeSize, dtypeSize); + int64_t devOffset = devIdx * dtypeSize; + int64_t hstOffset = hstIdx * dtypeSize; + if (hstOffset + dtypeSize > dstSize) { + return DebuggerErrno::ERROR_INVALID_VALUE; + } + std::copy(src + devOffset, src + devOffset + dtypeSize, + dst + hstOffset); } } } @@ -427,17 +442,17 @@ static DebuggerErrno FRAC_Z_TO_NCHW(AclTensorInfo& tensor) return FRAC_Z_TO_NCHW_WITH_GROUPS(tensor); } - AssertDim(tensor.hostShape, kDim4); + AssertDim(tensor.hostShape, DIM_4); AssertConsis(tensor); AllocTensorTransBuf(tensor); - auto n0 = tensor.deviceShape.at(fzN0); - auto ni = tensor.deviceShape.at(fzNi); - auto c0 = tensor.deviceShape.at(fzC0); - auto n = tensor.hostShape[kN]; - auto c = tensor.hostShape[kC]; - auto h = tensor.hostShape[kH]; - auto w = tensor.hostShape[kW]; + auto n0 = tensor.deviceShape.at(FZ_N0); + auto ni = tensor.deviceShape.at(FZ_NI); + auto c0 = tensor.deviceShape.at(FZ_C0); + auto n = tensor.hostShape[AXIS_N]; + auto c = tensor.hostShape[AXIS_C]; + auto h = tensor.hostShape[AXIS_H]; + auto w = tensor.hostShape[AXIS_W]; auto nc = ni * n0; auto ncc0 = nc * c0; auto wncc0 = w * ncc0; @@ -450,8 +465,9 @@ static DebuggerErrno FRAC_Z_TO_NCHW(AclTensorInfo& tensor) } const uint8_t* src = tensor.aclData; - uint8_t* dst = tensor.transBuf.data(); - auto dtypeSize = SizeOfAclDType(tensor); + auto dst = tensor.transBuf.begin(); + int64_t dtypeSize = static_cast(SizeOfAclDType(tensor)); + int64_t dstSize = static_cast(tensor.transBuf.size()); for (int64_t nIdx = 0; nIdx < n; nIdx++) { int64_t nHeadAddr = nIdx * chw; for (int64_t cIdx = 0; cIdx < c; cIdx++) { @@ -464,8 +480,13 @@ static DebuggerErrno FRAC_Z_TO_NCHW(AclTensorInfo& tensor) auto c0Idx = cIdx % c0; auto ncIdx = nIdx; auto srcIdx = c1Idx * hwncc0 + hIdx * wncc0 + wIdx * ncc0 + ncIdx * c0 + c0Idx; - /* 此处由偏移计算逻辑保障不会越界读写 */ - std::memcpy(dst + dstIdx * dtypeSize, src + srcIdx * dtypeSize, dtypeSize); + auto dstOffset = dstIdx * dtypeSize; + auto srcOffset = srcIdx * dtypeSize; + if (dstOffset + dtypeSize > dstSize) { + return DebuggerErrno::ERROR_INVALID_VALUE; + } + std::copy(src + srcOffset, src + srcOffset + dtypeSize, + dst + dstOffset); } } } @@ -475,7 +496,7 @@ static DebuggerErrno FRAC_Z_TO_NCHW(AclTensorInfo& tensor) static void TransShapeToHwNz(const AclShape &hostShape, AclShape& hwShape) { - if (hostShape.size() == kDim1) { + if (hostShape.size() == DIM_1) { hwShape.push_back(1); hwShape.push_back(1); hwShape.push_back(hostShape[0]); @@ -483,12 +504,12 @@ static void TransShapeToHwNz(const AclShape &hostShape, AclShape& hwShape) } auto size = hostShape.size(); int64_t times = 1; - for (size_t i = 0; i != size - kDim2; i++) { + for (size_t i = 0; i != size - DIM_2; i++) { times *= hostShape[i]; } hwShape.push_back(times); - hwShape.push_back(hostShape[size - kDim2]); - hwShape.push_back(hostShape[size - kDim1]); + hwShape.push_back(hostShape[size - DIM_2]); + hwShape.push_back(hostShape[size - DIM_1]); } static DebuggerErrno FRAC_NZ_TO_NCHW(AclTensorInfo& tensor) @@ -499,27 +520,32 @@ static DebuggerErrno FRAC_NZ_TO_NCHW(AclTensorInfo& tensor) AclShape hwShape; TransShapeToHwNz(tensor.hostShape, hwShape); auto times = hwShape.at(0); - auto h = hwShape.at(hwH); - auto w = hwShape.at(hwW); + auto h = hwShape.at(HW_H); + auto w = hwShape.at(HW_W); auto hw = h * w; auto shapeSize = tensor.deviceShape.size(); - if (shapeSize < kDim4) { + if (shapeSize < DIM_4) { LOG_WARNING(DebuggerErrno::ERROR_INVALID_VALUE, tensor + ": Invalid shape size."); return DebuggerErrno::ERROR_INVALID_VALUE; } - auto w1 = tensor.deviceShape[shapeSize - fnzW1]; - auto h1 = tensor.deviceShape[shapeSize - fnzH1]; - auto h0 = tensor.deviceShape[shapeSize - fnzH0]; - auto w0 = tensor.deviceShape[shapeSize - fnzW0]; + auto w1 = tensor.deviceShape[shapeSize - FNZ_W1]; + auto h1 = tensor.deviceShape[shapeSize - FNZ_H1]; + auto h0 = tensor.deviceShape[shapeSize - FNZ_H0]; + auto w0 = tensor.deviceShape[shapeSize - FNZ_W0]; auto h1h0w0 = h1 * h0 * w0; auto w1h1h0w0 = w1 * h1h0w0; + if (w0 == 0) { + LOG_WARNING(DebuggerErrno::ERROR_INVALID_VALUE, tensor + ": Invalid shape size."); + return DebuggerErrno::ERROR_INVALID_VALUE; + } auto numW1 = w / w0; const uint8_t* src = tensor.aclData; - uint8_t* dst = tensor.transBuf.data(); - auto dtypeSize = SizeOfAclDType(tensor); + auto dst = tensor.transBuf.begin(); + int64_t dtypeSize = static_cast(SizeOfAclDType(tensor)); + int64_t dstSize = static_cast(tensor.transBuf.size()); for (int64_t timesIdx = 0; timesIdx < times; timesIdx++) { auto timesHead = timesIdx * w1h1h0w0; @@ -531,8 +557,13 @@ static DebuggerErrno FRAC_NZ_TO_NCHW(AclTensorInfo& tensor) for (int64_t i = 0; i < w0; ++i) { int64_t srcIdx = h1h0Head + w1Idx * h1h0w0 + i; int64_t dstIdx = srcHHead + w1Idx * w0 + i; - /* 此处由偏移计算逻辑保障不会越界读写 */ - std::memcpy(dst + dstIdx * dtypeSize, src + srcIdx * dtypeSize, dtypeSize); + int64_t dstOffset = dstIdx * dtypeSize; + int64_t srcOffset = srcIdx * dtypeSize; + if (dstOffset + dtypeSize > dstSize) { + return DebuggerErrno::ERROR_INVALID_VALUE; + } + std::copy(src + srcOffset, src + srcOffset + dtypeSize, + dst + dstOffset); } } auto w1Head = numW1 * w0; @@ -540,8 +571,12 @@ static DebuggerErrno FRAC_NZ_TO_NCHW(AclTensorInfo& tensor) auto srcWIdx = w1Head + w0Idx; int64_t srcIdx = h1h0Head + numW1 * h1h0w0 + w0Idx; int64_t dstIdx = srcHHead + srcWIdx; - /* 此处由偏移计算逻辑保障不会越界读写 */ - std::memcpy(dst + dstIdx * dtypeSize, src + srcIdx * dtypeSize, dtypeSize); + int64_t dstOffset = dstIdx * dtypeSize; + int64_t srcOffset = srcIdx * dtypeSize; + if (dstOffset + dtypeSize > dstSize) { + return DebuggerErrno::ERROR_INVALID_VALUE; + } + std::copy(src + srcOffset, src + srcOffset + dtypeSize, dst + dstOffset); } } } @@ -550,16 +585,20 @@ static DebuggerErrno FRAC_NZ_TO_NCHW(AclTensorInfo& tensor) static DebuggerErrno NC1HWC0_TO_NCHW(AclTensorInfo& tensor) { - AssertDim(tensor.hostShape, kDim4); + AssertDim(tensor.hostShape, DIM_4); AssertConsis(tensor); AllocTensorTransBuf(tensor); - auto n = tensor.hostShape[kN]; - auto c = tensor.hostShape[kC]; - auto h = tensor.hostShape[kH]; - auto w = tensor.hostShape[kW]; - auto c1 = tensor.deviceShape[kDim1]; - auto c0 = tensor.deviceShape[kDim4]; + auto n = tensor.hostShape[AXIS_N]; + auto c = tensor.hostShape[AXIS_C]; + auto h = tensor.hostShape[AXIS_H]; + auto w = tensor.hostShape[AXIS_W]; + auto c1 = tensor.deviceShape[DIM_1]; + auto c0 = tensor.deviceShape[DIM_4]; + if (c0 == 0) { + LOG_WARNING(DebuggerErrno::ERROR_INVALID_VALUE, tensor + ": Invalid shape size."); + return DebuggerErrno::ERROR_INVALID_VALUE; + } auto hw = h * w; auto chw = c * hw; @@ -568,8 +607,9 @@ static DebuggerErrno NC1HWC0_TO_NCHW(AclTensorInfo& tensor) auto c1hwc0 = c1 * hwc0; const uint8_t* src = tensor.aclData; - uint8_t* dst = tensor.transBuf.data(); - auto dtypeSize = SizeOfAclDType(tensor); + auto dst = tensor.transBuf.begin(); + int64_t dtypeSize = static_cast(SizeOfAclDType(tensor)); + int64_t dstSize = static_cast(tensor.transBuf.size()); for (int64_t nIndex = 0; nIndex < n; nIndex++) { int64_t nHeadAddr = nIndex * chw; for (int64_t cIndex = 0; cIndex < c; cIndex++) { @@ -581,8 +621,13 @@ static DebuggerErrno NC1HWC0_TO_NCHW(AclTensorInfo& tensor) int64_t c1Index = cIndex / c0; int64_t c0Index = cIndex % c0; int64_t srcIdx = nIndex * c1hwc0 + c1Index * hwc0 + hIndex * wc0 + wIndex * c0 + c0Index; - /* 此处由偏移计算逻辑保障不会越界读写 */ - std::memcpy(dst + dstIdx * dtypeSize, src + srcIdx * dtypeSize, dtypeSize); + int64_t dstOffset = dstIdx * dtypeSize; + int64_t srcOffset = srcIdx * dtypeSize; + if (dstOffset + dtypeSize > dstSize) { + return DebuggerErrno::ERROR_INVALID_VALUE; + } + std::copy(src + srcOffset, src + srcOffset + dtypeSize, + dst + dstOffset); } } } @@ -592,17 +637,21 @@ static DebuggerErrno NC1HWC0_TO_NCHW(AclTensorInfo& tensor) static DebuggerErrno NDC1HWC0_TO_NCDHW(AclTensorInfo& tensor) { - AssertDim(tensor.hostShape, kDim5); + AssertDim(tensor.hostShape, DIM_5); AssertConsis(tensor); AllocTensorTransBuf(tensor); - auto n = tensor.hostShape[N_ncdhw]; - auto c = tensor.hostShape[C_ncdhw]; - auto d = tensor.hostShape[D_ncdhw]; - auto h = tensor.hostShape[H_ncdhw]; - auto w = tensor.hostShape[W_ncdhw]; - auto c1 = tensor.deviceShape[C1_ndc1hwc0]; - auto c0 = tensor.deviceShape[C0_ndc1hwc0]; + auto n = tensor.hostShape[N_NCDHW]; + auto c = tensor.hostShape[C_NCDHW]; + auto d = tensor.hostShape[D_NCDHW]; + auto h = tensor.hostShape[H_NCDHW]; + auto w = tensor.hostShape[W_NCDHW]; + auto c1 = tensor.deviceShape[C1_NDC1HWC0]; + auto c0 = tensor.deviceShape[C0_NDC1HWC0]; + if (c0 == 0) { + LOG_WARNING(DebuggerErrno::ERROR_INVALID_VALUE, tensor + ": Invalid shape size."); + return DebuggerErrno::ERROR_INVALID_VALUE; + } const int64_t cdhw = c * d * h * w; const int64_t dhw = d * h * w; @@ -613,8 +662,9 @@ static DebuggerErrno NDC1HWC0_TO_NCDHW(AclTensorInfo& tensor) const int64_t wc0 = w * c0; const uint8_t* src = tensor.aclData; - uint8_t* dst = tensor.transBuf.data(); - auto dtypeSize = SizeOfAclDType(tensor); + auto dst = tensor.transBuf.begin(); + int64_t dtypeSize = static_cast(SizeOfAclDType(tensor)); + int64_t dstSize = static_cast(tensor.transBuf.size()); for (int64_t nIndex = 0; nIndex < n; nIndex++) { int64_t nHead = nIndex * cdhw; for (int64_t cIndex = 0; cIndex < c; cIndex++) { @@ -629,8 +679,13 @@ static DebuggerErrno NDC1HWC0_TO_NCDHW(AclTensorInfo& tensor) int64_t c0Index = cIndex % c0; auto srcIdx = nIndex * dc1hwc0 + dIndex * c1hwc0 + c1Index * hwc0 + hIndex * wc0 + wIndex * c0 + c0Index; - /* 此处由偏移计算逻辑保障不会越界读写 */ - std::memcpy(dst + dstIdx * dtypeSize, src + srcIdx * dtypeSize, dtypeSize); + int64_t dstOffset = dstIdx * dtypeSize; + int64_t srcOffset = srcIdx * dtypeSize; + if (dstOffset + dtypeSize > dstSize) { + return DebuggerErrno::ERROR_INVALID_VALUE; + } + std::copy(src + srcOffset, src + srcOffset + dtypeSize, + dst + dstOffset); } } } @@ -641,14 +696,14 @@ static DebuggerErrno NDC1HWC0_TO_NCDHW(AclTensorInfo& tensor) static DebuggerErrno C1HWNCoC0_TO_NCHW(AclTensorInfo& tensor) { - AssertDim(tensor.hostShape, kDim4); + AssertDim(tensor.hostShape, DIM_4); AssertConsis(tensor); AllocTensorTransBuf(tensor); - auto n = tensor.hostShape[kN]; - auto c = tensor.hostShape[kC]; - auto h = tensor.hostShape[kH]; - auto w = tensor.hostShape[kW]; + auto n = tensor.hostShape[AXIS_N]; + auto c = tensor.hostShape[AXIS_C]; + auto h = tensor.hostShape[AXIS_H]; + auto w = tensor.hostShape[AXIS_W]; const int coIdx = 4; const int c0Idx = 5; auto co = tensor.deviceShape[coIdx]; @@ -656,8 +711,9 @@ static DebuggerErrno C1HWNCoC0_TO_NCHW(AclTensorInfo& tensor) auto cubeK = GetCubeSizeByType(tensor.dtype); const uint8_t* src = tensor.aclData; - uint8_t* dst = tensor.transBuf.data(); - auto dtypeSize = SizeOfAclDType(tensor); + auto dst = tensor.transBuf.begin(); + int64_t dtypeSize = static_cast(SizeOfAclDType(tensor)); + int64_t dstSize = static_cast(tensor.transBuf.size()); for (int64_t nIndex = 0; nIndex < n; nIndex++) { for (int64_t cIndex = 0; cIndex < c; cIndex++) { for (int64_t hIndex = 0; hIndex < h; hIndex++) { @@ -668,8 +724,13 @@ static DebuggerErrno C1HWNCoC0_TO_NCHW(AclTensorInfo& tensor) int64_t coIndex = c0Index; int64_t srcIdx = c1Index * h * w * n * co * c0 + hIndex * w * n * co * c0 + wIndex * n * co * c0 + nIndex * co * c0 + coIndex * c0 + c0Index; - /* 此处由偏移计算逻辑保障不会越界读写 */ - std::memcpy(dst + dstIdx * dtypeSize, src + srcIdx * dtypeSize, dtypeSize); + int64_t dstOffset = dstIdx * dtypeSize; + int64_t srcOffset = srcIdx * dtypeSize; + if (dstOffset + dtypeSize > dstSize) { + return DebuggerErrno::ERROR_INVALID_VALUE; + } + std::copy(src + srcOffset, src + srcOffset + dtypeSize, + dst + dstOffset); } } } @@ -684,17 +745,21 @@ static DebuggerErrno NC1HWC0_C04_TO_NCHW(AclTensorInfo& tensor) static DebuggerErrno FRAC_Z3D_TO_NCDHW(AclTensorInfo& tensor) { - AssertDim(tensor.hostShape, kDim5); + AssertDim(tensor.hostShape, DIM_5); AssertConsis(tensor); AllocTensorTransBuf(tensor); - auto n = tensor.hostShape[N_ncdhw]; - auto c = tensor.hostShape[C_ncdhw]; - auto d = tensor.hostShape[D_ncdhw]; - auto h = tensor.hostShape[H_ncdhw]; - auto w = tensor.hostShape[W_ncdhw]; - constexpr int kFZ3D_C0 = 3; - auto c0 = tensor.deviceShape[kFZ3D_C0]; + auto n = tensor.hostShape[N_NCDHW]; + auto c = tensor.hostShape[C_NCDHW]; + auto d = tensor.hostShape[D_NCDHW]; + auto h = tensor.hostShape[H_NCDHW]; + auto w = tensor.hostShape[W_NCDHW]; + constexpr int FZ3D_C0 = 3; + auto c0 = tensor.deviceShape[FZ3D_C0]; + if (c0 == 0) { + LOG_WARNING(DebuggerErrno::ERROR_INVALID_VALUE, tensor + ": Invalid shape size."); + return DebuggerErrno::ERROR_INVALID_VALUE; + } auto cube_k = GetCubeSizeByType(tensor.dtype); auto c1 = DivCeil(c, cube_k); constexpr int64_t kNiSize = 16; @@ -708,8 +773,9 @@ static DebuggerErrno FRAC_Z3D_TO_NCDHW(AclTensorInfo& tensor) auto cdhw = c * dhw; const uint8_t* src = tensor.aclData; - uint8_t* dst = tensor.transBuf.data(); - auto dtypeSize = SizeOfAclDType(tensor); + auto dst = tensor.transBuf.begin(); + int64_t dtypeSize = static_cast(SizeOfAclDType(tensor)); + int64_t dstSize = static_cast(tensor.transBuf.size()); for (int64_t nIdx = 0; nIdx < n; nIdx++) { int64_t nHead = nIdx * cdhw; for (int64_t cIdx = 0; cIdx < c; cIdx++) { @@ -725,8 +791,13 @@ static DebuggerErrno FRAC_Z3D_TO_NCDHW(AclTensorInfo& tensor) int64_t ncIdx = nIdx; int64_t srcIdx = dIdx * c1hwn1n0c0 + c1I * c1hwn1n0c0 + hIdx * wn1n0c0 + wI * n1n0c0 + ncIdx * c0 + c0I; - /* 此处由偏移计算逻辑保障不会越界读写 */ - std::memcpy(dst + dstIdx * dtypeSize, src + srcIdx * dtypeSize, dtypeSize); + int64_t dstOffset = dstIdx * dtypeSize; + int64_t srcOffset = srcIdx * dtypeSize; + if (dstOffset + dtypeSize > dstSize) { + return DebuggerErrno::ERROR_INVALID_VALUE; + } + std::copy(src + srcOffset, src + srcOffset + dtypeSize, + dst + dstOffset); } } } @@ -753,11 +824,11 @@ DebuggerErrno TransFormatD2H(AclTensorInfo& tensor) } } -static void TransBf16ToFp32(const uint8_t* input, size_t num, uint8_t* output, size_t bufferSize) +static DebuggerErrno TransBf16ToFp32(const uint8_t* input, size_t num, uint8_t* output, size_t bufferSize) { if (bufferSize < num * sizeof(float)) { LOG_ERROR(DebuggerErrno::ERROR_BUFFER_OVERFLOW, "Insufficient space for converting data from bf16 to fp32."); - return; + return DebuggerErrno::ERROR_BUFFER_OVERFLOW; } const DataUtils::BFloat16* in = reinterpret_cast(input); float* out = reinterpret_cast(output); @@ -765,13 +836,14 @@ static void TransBf16ToFp32(const uint8_t* input, size_t num, uint8_t* output, s for (size_t i = 0; i < num; i++) { out[i] = static_cast(in[i]); } + return DebuggerErrno::OK; } -static void TransInt4ToInt8(const uint8_t* input, size_t elemNums, uint8_t* output, size_t bufferSize) +static DebuggerErrno TransInt4ToInt8(const uint8_t* input, size_t elemNums, uint8_t* output, size_t bufferSize) { if (bufferSize < elemNums * sizeof(int8_t)) { LOG_ERROR(DebuggerErrno::ERROR_BUFFER_OVERFLOW, "Insufficient space for converting data from int4 to int8."); - return; + return DebuggerErrno::ERROR_BUFFER_OVERFLOW; } const int8_t *srcData = reinterpret_cast(input); int8_t *dstData = reinterpret_cast(output); @@ -811,34 +883,43 @@ static void TransInt4ToInt8(const uint8_t* input, size_t elemNums, uint8_t* outp ++dstData; ++srcData; } - return; + return DebuggerErrno::OK; } DebuggerErrno TransDtype(AclTensorInfo& tensor, AclDtype to) { - if (tensor.dtype == to) { return DebuggerErrno::OK; } tensor.oriDtype = tensor.dtype; std::vector buffer; - AssertConsis(tensor); + try { + AssertConsis(tensor); + } catch (const std::runtime_error& e) { + LOG_ERROR(DebuggerErrno::ERROR_INVALID_OPERATION, e.what()); + return DebuggerErrno::ERROR_INVALID_OPERATION; + } size_t bufferSize = EleNumOfTensor(tensor) * SizeOfAclDType(to); buffer.resize(bufferSize); const uint8_t* input = tensor.transBuf.empty() ? tensor.aclData : tensor.transBuf.data(); uint8_t* output = buffer.data(); + DebuggerErrno ret; if (tensor.dtype == AclDtype::DT_BF16 && to == AclDtype::DT_FLOAT) { - TransBf16ToFp32(input, EleNumOfTensor(tensor), output, bufferSize); + ret = TransBf16ToFp32(input, EleNumOfTensor(tensor), output, bufferSize); } else if (tensor.dtype == AclDtype::DT_INT4 && to == AclDtype::DT_INT8) { - TransInt4ToInt8(input, EleNumOfTensor(tensor), output, bufferSize); + ret = TransInt4ToInt8(input, EleNumOfTensor(tensor), output, bufferSize); } else { LOG_ERROR(DebuggerErrno::ERROR_UNKNOWN_TRANS, tensor + ": Trans " + DataUtils::GetDTypeString(tensor.dtype) + " to " + DataUtils::GetDTypeString(to) + " is not supported."); return DebuggerErrno::ERROR_UNKNOWN_TRANS; } + if (ret != DebuggerErrno::OK) { + return ret; + } + tensor.transBuf = std::move(buffer); tensor.dtype = to; return DebuggerErrno::OK; diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.hpp b/debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.h similarity index 83% rename from debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.hpp rename to debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.h index f2ac429a7f14370ea1721369c7f9089cb971bb6e..301da55ef7686255c12c8fb52dcfaf3e8314e3a8 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/AclTensor.h @@ -19,9 +19,9 @@ #include #include -#include "include/ErrorCode.hpp" +#include "include/ErrorCode.h" #include "proto/AclDumpMsg.pb.h" -#include "utils/DataUtils.hpp" +#include "utils/DataUtils.h" namespace MindStudioDebugger { @@ -29,12 +29,12 @@ using AclShape = DataUtils::TensorShape; using AclDtype = DataUtils::DataType; using AclFormat = DataUtils::TensorFormat; -constexpr uint8_t kDim1 = 1; -constexpr uint8_t kDim2 = 2; -constexpr uint8_t kDim3 = 3; -constexpr uint8_t kDim4 = 4; -constexpr uint8_t kDim5 = 5; -constexpr uint8_t kDim6 = 6; +constexpr uint8_t DIM_1 = 1; +constexpr uint8_t DIM_2 = 2; +constexpr uint8_t DIM_3 = 3; +constexpr uint8_t DIM_4 = 4; +constexpr uint8_t DIM_5 = 5; +constexpr uint8_t DIM_6 = 6; struct AclTensorInfo { std::string dumpPath; @@ -52,21 +52,24 @@ struct AclTensorInfo { bool dumpOriginData; std::vector transBuf; - std::string ToString() const { + std::string ToString() const + { return "AclTensor(path=" + dumpPath + ",dtype=" + DataUtils::GetDTypeString(dtype) + ",inout=" + inout + ")"; } }; -inline std::string operator+(const std::string& s, const AclTensorInfo& tensor) { +inline std::string operator+(const std::string& s, const AclTensorInfo& tensor) +{ return s + tensor.ToString(); } -inline std::string operator+(const AclTensorInfo& tensor, const std::string& s) { +inline std::string operator+(const AclTensorInfo& tensor, const std::string& s) +{ return tensor.ToString() + s; } namespace AclTensor { -size_t SizeOfTensor(const AclTensorInfo& tensor, bool host=true); +size_t SizeOfTensor(const AclTensorInfo& tensor, bool host = true); template AclTensorInfo ParseAttrsFromDumpData(const std::string &dumpPath, const uint8_t* data, const T& tensor, const std::string& io, uint32_t slot); diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.cpp b/debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.cpp index d4d74f1962222558c88c576b8ffbd8c474e152f2..6b51f6f28cee382e4b2928936387957d88f9f427 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.cpp @@ -16,10 +16,11 @@ #include -#include "base/ErrorInfos.hpp" -#include "base/DebuggerConfig.hpp" -#include "third_party/ACL/AclApi.hpp" -#include "PrecisionDebugger.hpp" +#include "base/ErrorInfosManager.h" +#include "base/DebuggerConfig.h" +#include "third_party/ACL/AclApi.h" +#include "core/mindspore/MSAclDumper.h" +#include "PrecisionDebugger.h" namespace MindStudioDebugger { @@ -83,12 +84,12 @@ int32_t PrecisionDebugger::Initialize(const std::string& framework, const std::s return ret; } - if(AscendCLApi::LoadAclApi() != DebuggerErrno::OK) { + if (AscendCLApi::LoadAclApi() != DebuggerErrno::OK) { return -1; } const DebuggerConfig& cfg = DebuggerConfig::GetInstance(); - for (auto iter = subDebuggers.begin(); iter != subDebuggers.end(); ) { + for (auto iter = subDebuggers.begin(); iter != subDebuggers.end();) { if (!(*iter)->Condition(cfg)) { iter = subDebuggers.erase(iter); } else { @@ -124,7 +125,7 @@ void PrecisionDebugger::Stop() } enable = false; - CALL_ACL_API(aclrtSynchronizeDevice); + CALL_ACL_API(AclrtSynchronizeDevice); for (auto task : subDebuggers) { task->OnStop(); @@ -133,25 +134,7 @@ void PrecisionDebugger::Stop() void PrecisionDebugger::Step() { - return Step(1); -} - -void PrecisionDebugger::Step(uint32_t step) -{ - DEBUG_FUNC_TRACE(); - if (!initialized) { - return; - } - - if (step > UINT32_MAX - curStep) { - throw std::runtime_error("Step over upper limit(4294967295)."); - } - curStep += step; - CALL_ACL_API(aclrtSynchronizeDevice); - - for (auto task : subDebuggers) { - task->OnStep(curStep); - } + MSAclDumper::GetInstance().Step(); } } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.hpp b/debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.h similarity index 92% rename from debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.hpp rename to debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.h index fbc22c016c40285a90a3de5989684098639256c9..939992d8151b620b4a6225ce912e97bd61a84cfa 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/PrecisionDebugger.h @@ -19,7 +19,7 @@ #include #include -#include "base/DebuggerConfig.hpp" +#include "base/DebuggerConfig.h" namespace MindStudioDebugger { @@ -43,9 +43,10 @@ protected: class PrecisionDebugger { public: - static PrecisionDebugger& GetInstance() { - static PrecisionDebugger instance_; - return instance_; + static PrecisionDebugger& GetInstance() + { + static PrecisionDebugger debuggerInstance; + return debuggerInstance; } int32_t Initialize(const std::string& framework, const std::string& cfgFile); diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.cpp b/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.cpp index 2d80ed3ce1ab11ee5ddf9bad18583a6813f32529..27f48412c690bbb5dafa0fdd31565f136718ab45 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.cpp @@ -16,15 +16,15 @@ #include -#include "base/ErrorInfos.hpp" -#include "base/DebuggerConfig.hpp" -#include "base/Environment.hpp" -#include "core/AclDumper.hpp" -#include "MSAclDumper.hpp" +#include "base/ErrorInfosManager.h" +#include "base/DebuggerConfig.h" +#include "base/Environment.h" +#include "core/AclDumper.h" +#include "MSAclDumper.h" namespace MindStudioDebugger { -void MSAclDumper::OnStepBegin(uint32_t device, uint32_t curStep, ExtArgs& args) +void MSAclDumper::OnStepBegin(uint32_t device, ExtArgs& args) { DEBUG_FUNC_TRACE(); if (!PrecisionDebugger::GetInstance().IsEnable()) { @@ -41,7 +41,7 @@ void MSAclDumper::OnStepBegin(uint32_t device, uint32_t curStep, ExtArgs& args) rank = static_cast(device); } - AclDumper::GetInstance().SetDump(rank, curStep, args); + AclDumper::GetInstance().SetDump(rank, msprobeStep, args); return; } @@ -51,6 +51,11 @@ void MSAclDumper::OnStepEnd(ExtArgs& args) AclDumper::GetInstance().FinalizeDump(args); } +void MSAclDumper::Step() +{ + msprobeStep++; +} + __attribute__((constructor)) void RegisterMSAclDumper() { MSAclDumper::GetInstance().Register(); diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.hpp b/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.h similarity index 78% rename from debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.hpp rename to debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.h index cd09bf51af0dac67065d51b8ce60c20f011cd585..3b4d53187faf1340dbdce0c182c47d58cdf52801 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MSAclDumper.h @@ -18,26 +18,29 @@ #include -#include "include/ExtArgs.hpp" -#include "core/PrecisionDebugger.hpp" +#include "include/ExtArgs.h" +#include "core/PrecisionDebugger.h" namespace MindStudioDebugger { class MSAclDumper : public PrecisionDbgTaskBase { public: - static MSAclDumper& GetInstance() { - static MSAclDumper instance_; - return instance_; + static MSAclDumper& GetInstance() + { + static MSAclDumper dumperInstance; + return dumperInstance; } std::string Name() const override {return "MindSpore AclDumper";} - bool Condition(const DebuggerConfig& cfg) const override { + bool Condition(const DebuggerConfig& cfg) const override + { return cfg.GetFramework() == DebuggerFramework::FRAMEWORK_MINDSPORE && cfg.GetDebugLevel() == DebuggerLevel::L2; } - void OnStepBegin(uint32_t device, uint32_t curStep, ExtArgs& args); + void OnStepBegin(uint32_t device, ExtArgs& args); void OnStepEnd(ExtArgs& args); + void Step(); private: MSAclDumper() = default; @@ -46,6 +49,7 @@ private: MSAclDumper& operator=(const MSAclDumper &obj) = delete; explicit MSAclDumper(MSAclDumper &&obj) = delete; MSAclDumper& operator=(MSAclDumper &&obj) = delete; + uint32_t msprobeStep{0}; }; } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.cpp b/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.cpp index 631ea7c4acf4666b911a3bb5f28a3c6cc4fe0d54..031b718737e9af877148b6e8c92192bd0c92fb47 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.cpp @@ -14,21 +14,22 @@ * limitations under the License. */ -#include "include/Macro.hpp" -#include "base/ErrorInfos.hpp" -#include "MindSporeTrigger.hpp" -#include "MSAclDumper.hpp" +#include "include/Macro.h" +#include "base/ErrorInfosManager.h" +#include "MSAclDumper.h" +#include "MindSporeTrigger.h" namespace MindStudioDebugger { bool MindSporeTrigger::stepBeginFlag = false; -void MindSporeTrigger::TriggerOnStepBegin(uint32_t device, uint32_t curStep, ExtArgs& args) +void MindSporeTrigger::TriggerOnStepBegin(uint32_t device, uint32_t /* curStep */, ExtArgs& args) { DEBUG_FUNC_TRACE(); CleanErrorInfoCache(); + + MSAclDumper::GetInstance().OnStepBegin(device, args); - MSAclDumper::GetInstance().OnStepBegin(device, curStep, args); stepBeginFlag = true; CleanErrorInfoCache(); diff --git a/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.hpp b/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.h similarity index 97% rename from debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.hpp rename to debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.h index 022e5d7d4c14a9771681840b967b2ec3aebb811b..d5048925bf58a1e4414b2983d796e598ac56c17b 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/core/mindspore/MindSporeTrigger.h @@ -18,7 +18,7 @@ #include -#include "include/ExtArgs.hpp" +#include "include/ExtArgs.h" namespace MindStudioDebugger { diff --git a/debug/accuracy_tools/msprobe/ccsrc/if/mindspore/MindSporeDbgHook.cpp b/debug/accuracy_tools/msprobe/ccsrc/if/mindspore/MindSporeDbgHook.cpp index 42f3a2e5b61d5da021b2ef7da4a7b88c6dc2abbb..2d744282d4eb2e741ae0e4afa7081a1a65738d61 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/if/mindspore/MindSporeDbgHook.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/if/mindspore/MindSporeDbgHook.cpp @@ -19,9 +19,9 @@ #include #include -#include "include/Macro.hpp" -#include "include/ExtArgs.hpp" -#include "core/mindspore/MindSporeTrigger.hpp" +#include "include/Macro.h" +#include "include/ExtArgs.h" +#include "core/mindspore/MindSporeTrigger.h" EXPORT_SYMBOL void MS_DbgOnStepBegin(uint32_t device, int32_t curStep, std::map exts) @@ -34,8 +34,11 @@ EXPORT_SYMBOL void MS_DbgOnStepBegin(uint32_t device, int32_t curStep, } /* mindspore使用了_GLIBCXX_USE_CXX11_ABI=0，为了解决CXX版本兼容问题，此处将string转char*使用 */ if (ext.first == static_cast(MindStudioDebugger::MindStudioExtensionArgs::ALL_KERNEL_NAMES)) { + if (ext.second == nullptr) { + continue; + } std::vector* ss = reinterpret_cast*>(ext.second); - strBuf = new const char*[(*ss).size() + 1]; + strBuf = new const char* [(*ss).size() + 1]; strBuf[(*ss).size()] = nullptr; size_t i = 0; for (std::string& s : *ss) { @@ -66,6 +69,4 @@ EXPORT_SYMBOL void MS_DbgOnStepEnd(std::map& exts) args[static_cast(ext.first)] = ext.second; } return MindStudioDebugger::MindSporeTrigger::TriggerOnStepEnd(args); -} - - +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/ccsrc/if/python/ACLDump.cpp b/debug/accuracy_tools/msprobe/ccsrc/if/python/ACLDump.cpp index 1c380ed3f505795eb622f7f401558f72a54db557..2bb73e34200216d77c5f884a343cd02b1250af7b 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/if/python/ACLDump.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/if/python/ACLDump.cpp @@ -18,37 +18,40 @@ #include #include -#include "base/ErrorInfos.hpp" -#include "core/AclDumper.hpp" -#include "utils/CPythonUtils.hpp" +#include "base/ErrorInfosManager.h" +#include "core/AclDumper.h" +#include "utils/CPythonUtils.h" namespace MindStudioDebugger { -static PyObject *CPythonKernelInitDump(PyObject *module, PyObject *args) { - PyGILState_STATE gstate = PyGILState_Ensure(); - KernelInitDump(); - PyGILState_Release(gstate); - Py_RETURN_NONE; +static PyObject *CPythonKernelInitDump(PyObject *module, PyObject *args) +{ + PyGILState_STATE gstate = PyGILState_Ensure(); + KernelInitDump(); + PyGILState_Release(gstate); + Py_RETURN_NONE; } -static PyObject *CPythonKernelSetDump(PyObject *module, PyObject *args) { - const char *path; - if (!PyArg_ParseTuple(args, "s", &path)) { +static PyObject *CPythonKernelSetDump(PyObject *module, PyObject *args) +{ + const char *path; + if (!PyArg_ParseTuple(args, "s", &path)) { LOG_ERROR(DebuggerErrno::ERROR_INVALID_VALUE, "npu set dump error, cfg_file must string"); return nullptr; - } - PyGILState_STATE gstate = PyGILState_Ensure(); - KernelSetDump(std::string(path)); - PyGILState_Release(gstate); - Py_RETURN_NONE; + } + PyGILState_STATE gstate = PyGILState_Ensure(); + KernelSetDump(std::string(path)); + PyGILState_Release(gstate); + Py_RETURN_NONE; } -static PyObject *CPythonKernelFinalizeDump(PyObject *module, PyObject *args) { - PyGILState_STATE gstate = PyGILState_Ensure(); - KernelFinalizeDump(); - PyGILState_Release(gstate); - Py_RETURN_NONE; +static PyObject *CPythonKernelFinalizeDump(PyObject *module, PyObject *args) +{ + PyGILState_STATE gstate = PyGILState_Ensure(); + KernelFinalizeDump(); + PyGILState_Release(gstate); + Py_RETURN_NONE; } static PyMethodDef DumpMethods[] = { diff --git a/debug/accuracy_tools/msprobe/ccsrc/if/python/ACLDump.hpp b/debug/accuracy_tools/msprobe/ccsrc/if/python/ACLDump.h similarity index 100% rename from debug/accuracy_tools/msprobe/ccsrc/if/python/ACLDump.hpp rename to debug/accuracy_tools/msprobe/ccsrc/if/python/ACLDump.h diff --git a/debug/accuracy_tools/msprobe/ccsrc/if/python/CPythonAgent.cpp b/debug/accuracy_tools/msprobe/ccsrc/if/python/CPythonAgent.cpp index 4b8fc03491e2c0792c3c707c272e7b587d60c7ad..e41243aa8d3c27b92c275dcd098e983083328d8e 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/if/python/CPythonAgent.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/if/python/CPythonAgent.cpp @@ -18,7 +18,7 @@ #include #include -#include "utils/CPythonUtils.hpp" +#include "utils/CPythonUtils.h" namespace MindStudioDebugger { @@ -29,8 +29,12 @@ PyDoc_STRVAR(CPythonAgentModuleDoc, static PyObject* CPythonAgentRegister(PyObject *module, PyObject *args) { + if (args == nullptr || !PyTuple_Check(args)) { + PyErr_SetString(PyExc_TypeError, "Expect a tuple."); + Py_RETURN_NONE; + } /* 预期2个参数，name和obj */ - if (args == nullptr || PyTuple_GET_SIZE(args) != 2) { + if (PyTuple_GET_SIZE(args) != 2) { PyErr_SetString(PyExc_TypeError, "\'register_context\' expects 2 arguments."); Py_RETURN_NONE; } @@ -56,7 +60,7 @@ static PyObject* CPythonAgentRegister(PyObject *module, PyObject *args) static PyObject* CPythonAgentUnRegister(PyObject *module, PyObject *obj) { CPythonUtils::PythonStringObject name(obj); - if(name.IsNone()) { + if (name.IsNone()) { PyErr_SetString(PyExc_TypeError, "\"name\" should be a string."); Py_RETURN_NONE; } @@ -68,7 +72,7 @@ static PyObject* CPythonAgentUnRegister(PyObject *module, PyObject *obj) static PyObject* CPythonAgentGetContext(PyObject *module, PyObject *obj) { CPythonUtils::PythonStringObject name(obj); - if(name.IsNone()) { + if (name.IsNone()) { PyErr_SetString(PyExc_TypeError, "\"name\" should be a string."); Py_RETURN_NONE; } diff --git a/debug/accuracy_tools/msprobe/ccsrc/if/python/CPythonAgent.hpp b/debug/accuracy_tools/msprobe/ccsrc/if/python/CPythonAgent.h similarity index 100% rename from debug/accuracy_tools/msprobe/ccsrc/if/python/CPythonAgent.hpp rename to debug/accuracy_tools/msprobe/ccsrc/if/python/CPythonAgent.h diff --git a/debug/accuracy_tools/msprobe/ccsrc/if/python/MsProbeIfPython.cpp b/debug/accuracy_tools/msprobe/ccsrc/if/python/MsProbeIfPython.cpp index a18c54a146f7d676d6b3c7f760e50f9e7eebe56c..fa3f65cc5fc4d211f9608cadc115601598232d05 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/if/python/MsProbeIfPython.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/if/python/MsProbeIfPython.cpp @@ -16,9 +16,9 @@ #include -#include "PrecisionDebuggerIfPython.hpp" -#include "CPythonAgent.hpp" -#include "ACLDump.hpp" +#include "PrecisionDebuggerIfPython.h" +#include "CPythonAgent.h" +#include "ACLDump.h" namespace MindStudioDebugger { @@ -27,7 +27,7 @@ PyDoc_STRVAR(MsProbeCModuleDoc, class _PrecisionDebugger: PrecisionDebugger in CXX \n\ class _DebuggerConfig: Configuration data of PrecisionDebugger \n\ class CPythonAgent: Used for front-end and back-end code interactions \n\ - \n\ + \n\ ..."); static struct PyModuleDef g_MsProbeCModule = { diff --git a/debug/accuracy_tools/msprobe/ccsrc/if/python/PrecisionDebuggerIfPython.cpp b/debug/accuracy_tools/msprobe/ccsrc/if/python/PrecisionDebuggerIfPython.cpp index da1cf3cf1c5d4c8894d0b12b5518657b5928a8d6..23e41db019dc8da8acce847245ca4e6bc41be67d 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/if/python/PrecisionDebuggerIfPython.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/if/python/PrecisionDebuggerIfPython.cpp @@ -18,8 +18,8 @@ #include #include -#include "utils/CPythonUtils.hpp" -#include "core/PrecisionDebugger.hpp" +#include "utils/CPythonUtils.h" +#include "core/PrecisionDebugger.h" namespace MindStudioDebugger { @@ -53,7 +53,6 @@ static int InitPrecisionDebugger(PyObject *self, PyObject *args, PyObject *kws) CPythonUtils::PythonDictObject kwArgs(kws); std::string framework = kwArgs.GetItem("framework"); std::string cfgFile = kwArgs.GetItem("config_path"); - if (PrecisionDebugger::GetInstance().Initialize(framework, cfgFile) != 0) { PyErr_SetString(PyExc_RuntimeError, "Failed to load config, read log for more details."); return -1; @@ -99,20 +98,9 @@ static PyObject* PrecisionDebuggerStop(PyObject *self) Py_RETURN_NONE; } -static PyObject* PrecisionDebuggerStep(PyObject *self, PyObject *args) +static PyObject* PrecisionDebuggerStep(PyObject *self) { - if (args == nullptr || PyTuple_GET_SIZE(args) == 0) { - PrecisionDebugger::GetInstance().Step(); - Py_RETURN_NONE; - } - - PyObject* increment = PyTuple_GetItem(args, 0); - if (!PyLong_Check(increment)) { - PyErr_SetString(PyExc_TypeError, "\'step\' should be a int."); - Py_RETURN_NONE; - } - - PrecisionDebugger::GetInstance().Step(PyLong_AsUnsignedLong(increment)); + PrecisionDebugger::GetInstance().Step(); Py_RETURN_NONE; } @@ -126,7 +114,7 @@ PyDoc_STRVAR(StepDoc, static PyMethodDef PrecisionDebuggerMethods[] = { {"start", reinterpret_cast(PrecisionDebuggerStart), METH_NOARGS, StartDoc}, {"stop", reinterpret_cast(PrecisionDebuggerStop), METH_NOARGS, StopDoc}, - {"step", reinterpret_cast(PrecisionDebuggerStep), METH_VARARGS, StepDoc}, + {"step", reinterpret_cast(PrecisionDebuggerStep), METH_NOARGS, StepDoc}, {nullptr, nullptr, 0, nullptr} }; @@ -184,5 +172,4 @@ PyTypeObject* GetPyPrecisionDebuggerType() } return &PyPrecisionDebuggerType; } - } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/ccsrc/if/python/PrecisionDebuggerIfPython.hpp b/debug/accuracy_tools/msprobe/ccsrc/if/python/PrecisionDebuggerIfPython.h similarity index 100% rename from debug/accuracy_tools/msprobe/ccsrc/if/python/PrecisionDebuggerIfPython.hpp rename to debug/accuracy_tools/msprobe/ccsrc/if/python/PrecisionDebuggerIfPython.h diff --git a/debug/accuracy_tools/msprobe/ccsrc/include/ErrorCode.hpp b/debug/accuracy_tools/msprobe/ccsrc/include/ErrorCode.h similarity index 100% rename from debug/accuracy_tools/msprobe/ccsrc/include/ErrorCode.hpp rename to debug/accuracy_tools/msprobe/ccsrc/include/ErrorCode.h diff --git a/debug/accuracy_tools/msprobe/ccsrc/include/ExtArgs.hpp b/debug/accuracy_tools/msprobe/ccsrc/include/ExtArgs.h similarity index 100% rename from debug/accuracy_tools/msprobe/ccsrc/include/ExtArgs.hpp rename to debug/accuracy_tools/msprobe/ccsrc/include/ExtArgs.h diff --git a/debug/accuracy_tools/msprobe/ccsrc/include/Macro.hpp b/debug/accuracy_tools/msprobe/ccsrc/include/Macro.h similarity index 100% rename from debug/accuracy_tools/msprobe/ccsrc/include/Macro.hpp rename to debug/accuracy_tools/msprobe/ccsrc/include/Macro.h diff --git a/debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.cpp b/debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.cpp index 1636c6998d9096b62e9a7f281c7e5ac1b4de4818..c79f2820a9f693afb93cdf68b3f2f7c751d7e389 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.cpp @@ -18,30 +18,30 @@ #include #include -#include "base/ErrorInfos.hpp" -#include "AclApi.hpp" +#include "base/ErrorInfosManager.h" +#include "AclApi.h" namespace MindStudioDebugger { namespace AscendCLApi { using namespace MindStudioDebugger; -constexpr const char* kLibAscendclName = "libascendcl.so"; -constexpr const char* kLibMSAscendName = "libmindspore_ascend.so.2"; +constexpr const char* LIB_ASCEND_CL_NAME = "libascendcl.so"; +constexpr const char* LIB_MS_ASCEND_NAME = "libmindspore_ascend.so.2"; -using aclInitFuncType = aclError (*)(const char *); -using aclmdlInitDumpFuncType = aclError (*)(); -using aclmdlSetDumpFuncType = aclError (*)(const char *); -using aclmdlFinalizeDumpFuncType = aclError (*)(); -using acldumpRegCallbackFuncType = aclError (*)(AclDumpCallbackFuncType, int32_t); -using aclrtSynchronizeDeviceFuncType = aclError (*)(); +using AclInitFuncType = aclError (*)(const char *); +using AclmdlInitDumpFuncType = aclError (*)(); +using AclmdlSetDumpFuncType = aclError (*)(const char *); +using AclmdlFinalizeDumpFuncType = aclError (*)(); +using AcldumpRegCallbackFuncType = aclError (*)(AclDumpCallbackFuncType, int32_t); +using AclrtSynchronizeDeviceFuncType = aclError (*)(); -static aclInitFuncType aclInitFunc = nullptr; -static aclmdlInitDumpFuncType aclmdlInitDumpFunc = nullptr; -static aclmdlSetDumpFuncType aclmdlSetDumpFunc = nullptr; -static aclmdlFinalizeDumpFuncType aclmdlFinalizeDumpFunc = nullptr; -static acldumpRegCallbackFuncType acldumpRegCallbackFunc = nullptr; -static aclrtSynchronizeDeviceFuncType aclrtSynchronizeDeviceFunc = nullptr; +static AclInitFuncType g_aclInitFunc = nullptr; +static AclmdlInitDumpFuncType g_aclmdlInitDumpFunc = nullptr; +static AclmdlSetDumpFuncType g_aclmdlSetDumpFunc = nullptr; +static AclmdlFinalizeDumpFuncType g_aclmdlFinalizeDumpFunc = nullptr; +static AcldumpRegCallbackFuncType g_acldumpRegCallbackFunc = nullptr; +static AclrtSynchronizeDeviceFuncType g_aclrtSynchronizeDeviceFunc = nullptr; DebuggerErrno LoadAclApi() { @@ -52,7 +52,7 @@ DebuggerErrno LoadAclApi() return DebuggerErrno::OK; } - hLibAscendcl = dlopen(kLibAscendclName, RTLD_LAZY); + hLibAscendcl = dlopen(LIB_ASCEND_CL_NAME, RTLD_LAZY | RTLD_NOLOAD); if (hLibAscendcl == nullptr) { LOG_ERROR(DebuggerErrno::ERROR_DEPENDENCY_NOT_FIND, "Failed to search libascendcl.so." + std::string(dlerror())); @@ -60,11 +60,11 @@ DebuggerErrno LoadAclApi() } static const std::map functionMap = { - {"aclInit", reinterpret_cast(&aclInitFunc)}, - {"aclmdlInitDump", reinterpret_cast(&aclmdlInitDumpFunc)}, - {"aclmdlSetDump", reinterpret_cast(&aclmdlSetDumpFunc)}, - {"aclmdlFinalizeDump", reinterpret_cast(&aclmdlFinalizeDumpFunc)}, - {"aclrtSynchronizeDevice", reinterpret_cast(&aclrtSynchronizeDeviceFunc)}, + {"aclInit", reinterpret_cast(&g_aclInitFunc)}, + {"aclmdlInitDump", reinterpret_cast(&g_aclmdlInitDumpFunc)}, + {"aclmdlSetDump", reinterpret_cast(&g_aclmdlSetDumpFunc)}, + {"aclmdlFinalizeDump", reinterpret_cast(&g_aclmdlFinalizeDumpFunc)}, + {"aclrtSynchronizeDevice", reinterpret_cast(&g_aclrtSynchronizeDeviceFunc)}, }; for (auto& iter : functionMap) { @@ -83,15 +83,15 @@ DebuggerErrno LoadAclApi() } /* 规避adump的bug，mindspore场景优先使用libmindspore_ascend.so中的符号 */ - void* handler = dlopen(kLibMSAscendName, RTLD_LAZY); - std::string libName = kLibMSAscendName; + void* handler = dlopen(LIB_MS_ASCEND_NAME, RTLD_LAZY | RTLD_NOLOAD); + std::string libName = LIB_MS_ASCEND_NAME; if (handler == nullptr) { handler = hLibAscendcl; - libName = kLibAscendclName; + libName = LIB_ASCEND_CL_NAME; } - acldumpRegCallbackFunc = reinterpret_cast(dlsym(handler, "acldumpRegCallback")); - if (acldumpRegCallbackFunc == nullptr) { + g_acldumpRegCallbackFunc = reinterpret_cast(dlsym(handler, "acldumpRegCallback")); + if (g_acldumpRegCallbackFunc == nullptr) { LOG_ERROR(DebuggerErrno::ERROR_DEPENDENCY_NOT_FIND, "Failed to load function acldumpRegCallback from " + libName + "."); } @@ -104,53 +104,53 @@ DebuggerErrno LoadAclApi() return DebuggerErrno::OK; } -aclError ACLAPI_aclInit(const char* cfg) +aclError AclApiAclInit(const char* cfg) { - if (aclInitFunc == nullptr) { + if (g_aclInitFunc == nullptr) { throw std::runtime_error("API aclInit does not have a definition."); } - return aclInitFunc(cfg); + return g_aclInitFunc(cfg); } -aclError ACLAPI_aclmdlInitDump() +aclError AclApiAclmdlInitDump() { - if (aclmdlInitDumpFunc == nullptr) { + if (g_aclmdlInitDumpFunc == nullptr) { throw std::runtime_error("API aclmdlInitDump does not have a definition."); } - return aclmdlInitDumpFunc(); + return g_aclmdlInitDumpFunc(); } -aclError ACLAPI_aclmdlSetDump(const char* cfg) +aclError AclApiAclmdlSetDump(const char* cfg) { - if (aclmdlSetDumpFunc == nullptr) { + if (g_aclmdlSetDumpFunc == nullptr) { throw std::runtime_error("API aclmdlSetDump does not have a definition."); } - return aclmdlSetDumpFunc(cfg); + return g_aclmdlSetDumpFunc(cfg); } -aclError ACLAPI_aclmdlFinalizeDump() +aclError AclApiAclmdlFinalizeDump() { - if (aclmdlFinalizeDumpFunc == nullptr) { + if (g_aclmdlFinalizeDumpFunc == nullptr) { throw std::runtime_error("API aclmdlFinalizeDump does not have a definition."); } - return aclmdlFinalizeDumpFunc(); + return g_aclmdlFinalizeDumpFunc(); } -aclError ACLAPI_acldumpRegCallback(AclDumpCallbackFuncType messageCallback, int32_t flag) +aclError AclApiAcldumpRegCallback(AclDumpCallbackFuncType messageCallback, int32_t flag) { - if (acldumpRegCallbackFunc == nullptr) { + if (g_acldumpRegCallbackFunc == nullptr) { throw std::runtime_error("API acldumpRegCallback does not have a definition."); } - return acldumpRegCallbackFunc(messageCallback, flag); + return g_acldumpRegCallbackFunc(messageCallback, flag); } -aclError ACLAPI_aclrtSynchronizeDevice() +aclError AclApiAclrtSynchronizeDevice() { - if (aclrtSynchronizeDeviceFunc == nullptr) { + if (g_aclrtSynchronizeDeviceFunc == nullptr) { throw std::runtime_error("API aclrtSynchronizeDevice does not have a definition."); } - return aclrtSynchronizeDeviceFunc(); + return g_aclrtSynchronizeDeviceFunc(); } -} +} } diff --git a/debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.hpp b/debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.h similarity index 78% rename from debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.hpp rename to debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.h index 731ae2e2caacaa345605ec572c8dcd6dba091488..366826fac943e622da58d8a136e66f18253b40c1 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/third_party/ACL/AclApi.h @@ -18,24 +18,23 @@ #include -#include "include/ErrorCode.hpp" +#include "include/ErrorCode.h" extern "C" { - -typedef int aclError; +using aclError = int; constexpr int ACL_SUCCESS = 0; constexpr int ACL_ERROR_NONE = 0; constexpr int ACL_ERROR_REPEAT_INITIALIZE = 100002; #define ACL_DUMP_MAX_FILE_PATH_LENGTH 4096 -typedef struct acldumpChunk { +typedef struct AclDumpChunk { char fileName[ACL_DUMP_MAX_FILE_PATH_LENGTH]; // 待落盘的Dump数据文件名，ACL_DUMP_MAX_FILE_PATH_LENGTH表示文件名最大长度，当前为4096 uint32_t bufLen; // dataBuf数据长度，单位Byte uint32_t isLastChunk; // 标识Dump数据是否为最后一个分片，0表示不是最后一个分片，1表示最后一个分片 int64_t offset; // Dump数据文件内容的偏移，其中-1表示文件追加内容 int32_t flag; // 预留Dump数据标识，当前数据无标识 uint8_t dataBuf[0]; // Dump数据的内存地址 -} acldumpChunk; +} AclDumpChunk; } @@ -44,16 +43,16 @@ namespace AscendCLApi { DebuggerErrno LoadAclApi(); -using AclDumpCallbackFuncType = int32_t (*)(const acldumpChunk*, int32_t); -aclError ACLAPI_aclInit(const char* cfg); -aclError ACLAPI_aclmdlInitDump(); -aclError ACLAPI_aclmdlSetDump(const char* cfg); -aclError ACLAPI_aclmdlFinalizeDump(); -aclError ACLAPI_acldumpRegCallback(AclDumpCallbackFuncType messageCallback, int32_t flag); +using AclDumpCallbackFuncType = int32_t (*)(const AclDumpChunk*, int32_t); +aclError AclApiAclInit(const char* cfg); +aclError AclApiAclmdlInitDump(); +aclError AclApiAclmdlSetDump(const char* cfg); +aclError AclApiAclmdlFinalizeDump(); +aclError AclApiAcldumpRegCallback(AclDumpCallbackFuncType messageCallback, int32_t flag); -aclError ACLAPI_aclrtSynchronizeDevice(); +aclError AclApiAclrtSynchronizeDevice(); -#define CALL_ACL_API(func, ...) MindStudioDebugger::AscendCLApi::ACLAPI_##func(__VA_ARGS__) +#define CALL_ACL_API(func, ...) MindStudioDebugger::AscendCLApi::AclApi##func(__VA_ARGS__) } } diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.cpp b/debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.cpp index fd944f62db4ff728d1aa2c5d1d5ff818bd5dcf62..932a2adbc74fd71091c629adbe289c779ca8288e 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.cpp @@ -18,7 +18,7 @@ #include #include -#include "CPythonUtils.hpp" +#include "CPythonUtils.h" namespace MindStudioDebugger { namespace CPythonUtils { @@ -77,7 +77,6 @@ PythonObject PythonObject::From(const uint32_t& input) PythonObject PythonObject::From(const double& input) { return PythonNumberObject::From(input); - } PythonObject PythonObject::From(const std::string& input) { @@ -108,7 +107,7 @@ int32_t PythonObject::To(uint32_t& output) const if (!PyLong_Check(ptr)) { return -1; } - output = static_cast(PyLong_AsUnsignedLong(ptr)); + output = static_cast(PyLong_AsUnsignedLong(ptr)); return 0; } @@ -155,7 +154,7 @@ PythonObject PythonObject::Get(const std::string& name, bool ignore) const return ret; } -PythonObject PythonObject::Call(bool ignore) +PythonObject PythonObject::Call(bool ignore) noexcept { if (!PyCallable_Check(ptr)) { if (!ignore) { @@ -173,7 +172,7 @@ PythonObject PythonObject::Call(bool ignore) return ret; } -PythonObject PythonObject::Call(PythonTupleObject& args, bool ignore) +PythonObject PythonObject::Call(PythonTupleObject& args, bool ignore) noexcept { if (!PyCallable_Check(ptr)) { if (!ignore) { @@ -191,7 +190,7 @@ PythonObject PythonObject::Call(PythonTupleObject& args, bool ignore) return ret; } -PythonObject PythonObject::Call(PythonTupleObject& args, PythonDictObject& kwargs, bool ignore) +PythonObject PythonObject::Call(PythonTupleObject& args, PythonDictObject& kwargs, bool ignore) noexcept { if (!PyCallable_Check(ptr)) { if (!ignore) { @@ -203,7 +202,7 @@ PythonObject PythonObject::Call(PythonTupleObject& args, PythonDictObject& kwarg if (args.IsNone() || kwargs.IsNone()) { if (!ignore) { PyErr_SetString(PyExc_TypeError, "Call python object with invalid parameters."); - } + } return PythonObject(); } @@ -227,10 +226,9 @@ PythonObject PythonObject::GetGlobal(const std::string& name, bool ignore) } return PythonObject(PyDict_GetItemString(globals, name.c_str())); - } -PythonObject PythonObject::Import(const std::string& name, bool ignore) +PythonObject PythonObject::Import(const std::string& name, bool ignore) noexcept { PyObject* m = PyImport_ImportModule(name.c_str()); if (m == nullptr) { @@ -483,7 +481,7 @@ PythonTupleObject::PythonTupleObject() : PythonObject() PythonTupleObject::PythonTupleObject(PyObject* o) : PythonObject() { - if (!PyTuple_Check(o)) { + if (!o || !PyTuple_Check(o)) { return; } diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.hpp b/debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.h similarity index 91% rename from debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.hpp rename to debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.h index 40ebcb1dafd505fd7dfa3bda1c2c1609cb60297a..db5153139362c07548e39fc6d17047673e2a4dd6 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/CPythonUtils.h @@ -40,14 +40,14 @@ namespace CPythonUtils { * | tuple | PythonTupleObject | * | dict | PythonDictObject | * ------------------------------------------- - * + * * 创建对象的方式： * 1、通过原生PyObject*类型创建，PythonObject生命周期内会持有原生对象的一个引用 * 2、通过From方法从c++对象创建 * 3、通过GetGlobal、Import等方法从解释器上下文获取 * 4、通过GetRegisteredPyObj获取到上下文的python对象 * 5、通过已有PythonObject对象的Get、GetItem等方法获取子对象 - * + * * 对象转换： * 1、对于转换成PyObject*、bool、string的场景，支持隐式转换 * 2、对于非通用类型转换，调用To方法，返回0表示成功 @@ -56,7 +56,7 @@ namespace CPythonUtils { * python维度支持bool()的都可以转bool（即并非只有bool类型支持转换，下同） * 支持str()的都可以转string * 可迭代对象(且元素支持转换)都可以转vector - * + * * 对象传递： * 1、子类可以安全传递或拷贝给PythonObject对象 * 2、PythonObject传给子类时，若类型匹配，可以安全转递，否则会转为None @@ -81,7 +81,8 @@ PythonObject GetRegisteredPyObj(const std::string& name); class PythonObject { public: - PythonObject() { + PythonObject() + { Py_INCREF(Py_None); ptr = Py_None; } @@ -91,19 +92,21 @@ public: } Py_XINCREF(ptr); } - ~PythonObject() { + ~PythonObject() + { Py_XDECREF(ptr); } explicit PythonObject(const PythonObject &obj) : PythonObject(static_cast(obj)) {} - PythonObject& operator=(const PythonObject &obj) { + PythonObject& operator=(const PythonObject &obj) + { SetPtr(static_cast(obj)); return *this; } /* 获取全局对象 */ - static PythonObject GetGlobal(const std::string& name, bool ignore=true); + static PythonObject GetGlobal(const std::string& name, bool ignore = true); /* 获取模块对象；若其还未加载至缓存，则加载一遍 */ - static PythonObject Import(const std::string& name, bool ignore=true); + static PythonObject Import (const std::string& name, bool ignore = true) noexcept; /* From/To转换，统一放一份在基类，用于遍历迭代器等场景 */ static PythonObject From(const PythonObject& input); @@ -136,17 +139,19 @@ public: bool IsCallable() const {return PyCallable_Check(ptr);} /* 用于调用可调用对象，相当于python代码中的obj()，为了简单只实现了args+kwargs参数形式 */ - PythonObject Call(bool ignore=true); - PythonObject Call(PythonTupleObject& args, bool ignore=true); - PythonObject Call(PythonTupleObject& args, PythonDictObject& kwargs, bool ignore=true); + PythonObject Call(bool ignore = true) noexcept; + PythonObject Call(PythonTupleObject& args, bool ignore = true) noexcept; + PythonObject Call(PythonTupleObject& args, PythonDictObject& kwargs, bool ignore = true) noexcept; /* 用于获取对象属性，相当于python代码中的obj.xx */ - PythonObject Get(const std::string& name, bool ignore=true) const; - PythonObject& NewRef() { + PythonObject Get(const std::string& name, bool ignore = true) const; + PythonObject& NewRef() + { Py_XINCREF(ptr); return *this; } - std::string ToString() const { + std::string ToString() const + { std::string ret; if (To(ret) == 0) { return ret; @@ -156,21 +161,24 @@ public: operator PyObject*() const {return ptr;} operator bool() const {return static_cast(PyObject_IsTrue(ptr));} - operator std::string() const { + operator std::string() const + { return ToString(); } - PythonObject operator()(bool ignore=true) {return Call(ignore);} - PythonObject operator()(PythonTupleObject& args, bool ignore=true) {return Call(args, ignore);} - PythonObject operator()(PythonTupleObject& args, PythonDictObject& kwargs, bool ignore=true) { + PythonObject operator()(bool ignore = true) {return Call(ignore);} + PythonObject operator()(PythonTupleObject& args, bool ignore = true) {return Call(args, ignore);} + PythonObject operator()(PythonTupleObject& args, PythonDictObject& kwargs, bool ignore = true) + { return Call(args, kwargs, ignore); } protected: - void SetPtr(PyObject* o) { + void SetPtr(PyObject* o) + { Py_XDECREF(ptr); if (o == nullptr) { o = Py_None; - } + } Py_INCREF(o); ptr = o; } @@ -220,11 +228,11 @@ public: size_t Size() const; template - PythonListObject& Append(T value, bool ignore=true); - PythonObject GetItem(size_t pos, bool ignore=true); - PythonListObject& SetItem(size_t pos, PythonObject& item, bool ignore=true); - PythonListObject& Insert(int64_t pos, PythonObject& item, bool ignore=true); - PythonTupleObject ToTuple(bool ignore=true); + PythonListObject& Append(T value, bool ignore = true); + PythonObject GetItem(size_t pos, bool ignore = true); + PythonListObject& SetItem(size_t pos, PythonObject& item, bool ignore = true); + PythonListObject& Insert(int64_t pos, PythonObject& item, bool ignore = true); + PythonTupleObject ToTuple(bool ignore = true); }; class PythonTupleObject : public PythonObject { @@ -236,7 +244,7 @@ public: static PythonTupleObject From(const std::vector& input); size_t Size() const; - PythonObject GetItem(size_t pos, bool ignore=true); + PythonObject GetItem(size_t pos, bool ignore = true); }; class PythonDictObject : public PythonObject { @@ -248,11 +256,11 @@ public: static PythonDictObject From(const std::map& input); template - PythonDictObject& Add(T1 key, T2 value, bool ignore=true); + PythonDictObject& Add(T1 key, T2 value, bool ignore = true); template - PythonDictObject& Delete(T key, bool ignore=true); + PythonDictObject& Delete(T key, bool ignore = true); template - PythonObject GetItem(T key, bool ignore=true); + PythonObject GetItem(T key, bool ignore = true); }; /**************************************************************************************************/ diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.cpp b/debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.cpp index c2d7df85294f7c96f0fe1a1b9458dfd2ad2e502c..c6744b68e67285e3cabebde1462f3305d2f863a4 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.cpp @@ -21,19 +21,21 @@ #include #include -#include "DataUtils.hpp" +#include "DataUtils.h" namespace MindStudioDebugger { namespace DataUtils { -int64_t SizeToS64(size_t v) { +int64_t SizeToS64(size_t v) +{ if (v > static_cast(INT64_MAX)) { throw std::runtime_error("Value " + std::to_string(v) + "exceeds the maximum value of int64."); } return static_cast(v); } -std::string U64ToHexString(uint64_t v) { +std::string U64ToHexString(uint64_t v) +{ std::stringstream ss; ss << "0x" << std::hex << std::uppercase << v; return std::move(ss.str()); @@ -42,28 +44,33 @@ std::string U64ToHexString(uint64_t v) { BFloat16::BFloat16(float f32) { if (std::isnan(f32)) { - value_ = BFloat16::nan_value; + value_ = BFloat16::NAN_VALUE; } else { + constexpr uint8_t offsetSize = 16; union { - uint32_t U32; - float F32; + uint32_t u32Value; + float f32Value; }; - F32 = f32; - uint32_t rounding_bias = ((U32 >> 16) & 1) + UINT32_C(0x7FFF); - value_ = static_cast((U32 + rounding_bias) >> 16); + f32Value = f32; + uint32_t rounding_bias = ((u32Value >> offsetSize) & 1) + UINT32_C(0x7FFF); + value_ = static_cast((u32Value + rounding_bias) >> offsetSize); } } BFloat16::operator float() const { - float f32 = 0; - uint32_t tmp = value_; - tmp <<= 16; - std::memcpy(&f32, &tmp, sizeof(f32)); + /* 为了兼容性，不要用c++20的bit_cast */ + constexpr uint8_t offsetSize = 16; + union { + float f32; + uint32_t ui32; + }; + ui32 = static_cast(value_); + ui32 <<= offsetSize; // 将ui32左移16位 return f32; } -const static std::unordered_map kTypeSizeMap = { +constexpr std::pair TYPE_SIZE_ARRAY[] = { {DataType::DT_BOOL, 1}, {DataType::DT_INT8, 1}, {DataType::DT_UINT8, 1}, @@ -83,15 +90,16 @@ const static std::unordered_map kTypeSizeMap = { size_t SizeOfDType(DataType type) { - auto it = kTypeSizeMap.find(type); - if (it == kTypeSizeMap.end()) { - return 0; + for (const auto& pair : TYPE_SIZE_ARRAY) { + if (pair.first == type) { + return pair.second; + } } - return it->second; + return 0; } -constexpr auto kOpDType_UNKNOWN = "UNKNOWN"; -const static std::unordered_map kDDTypeToStringMap = { +constexpr auto OP_DTYPE_UNKNOWN = "UNKNOWN"; +const std::pair DTYPE_TO_STRING_ARRAY[] = { {DataType::DT_UNDEFINED, "UNDEFINED"}, {DataType::DT_FLOAT, "FLOAT"}, {DataType::DT_FLOAT16, "FLOAT16"}, @@ -128,15 +136,16 @@ const static std::unordered_map kDDTypeToStringMap = { std::string GetDTypeString(DataType dtype) { - auto it = kDDTypeToStringMap.find(dtype); - if (it != kDDTypeToStringMap.end()) { - return it->second; + for (const auto& pair : DTYPE_TO_STRING_ARRAY) { + if (pair.first == dtype) { + return std::string(pair.second); + } } - return kOpDType_UNKNOWN; + return OP_DTYPE_UNKNOWN; } -constexpr auto kOpFormat_UNKNOWN = "UNKNOWN"; -const static std::unordered_map kFormatToStringMap = { +constexpr auto OP_FORMAT_UNKNOWN = "UNKNOWN"; +const std::pair FORMAT_TO_STRING_ARRAY[] = { {TensorFormat::FORMAT_NCHW, "NCHW"}, {TensorFormat::FORMAT_NHWC, "NHWC"}, {TensorFormat::FORMAT_ND, "ND"}, @@ -162,7 +171,7 @@ const static std::unordered_map kFormatToStringMap = {TensorFormat::FORMAT_HASHTABLE_LOOKUP_VALUE, "HASHTABLE_LOOKUP_VALUE"}, {TensorFormat::FORMAT_HASHTABLE_LOOKUP_OUTPUT, "HASHTABLE_LOOKUP_OUTPUT"}, {TensorFormat::FORMAT_HASHTABLE_LOOKUP_HITS, "HASHTABLE_LOOKUP_HITS"}, - {TensorFormat::FORMAT_C1HWNCoC0, "C1HWNCoC0"}, + {TensorFormat::FORMAT_C1HWNCOC0, "C1HWNCoC0"}, {TensorFormat::FORMAT_MD, "MD"}, {TensorFormat::FORMAT_NDHWC, "NDHWC"}, {TensorFormat::FORMAT_FRACTAL_ZZ, "FRACTAL_ZZ"}, @@ -191,11 +200,12 @@ const static std::unordered_map kFormatToStringMap = std::string GetFormatString(TensorFormat fmt) { - auto it = kFormatToStringMap.find(fmt); - if (it != kFormatToStringMap.end()) { - return it->second; + for (const auto& pair : FORMAT_TO_STRING_ARRAY) { + if (pair.first == fmt) { + return std::string(pair.second); + } } - return kOpFormat_UNKNOWN; + return OP_FORMAT_UNKNOWN; } std::string GetShapeString(const TensorShape& shape) diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.hpp b/debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.h similarity index 90% rename from debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.hpp rename to debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.h index f58e15a8c77719f62ddeef8ebbcd25a5b5ebf624..35f9ae4f242f8575ea98d86a3da95b381c63fbef 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/DataUtils.h @@ -24,11 +24,11 @@ namespace MindStudioDebugger { namespace DataUtils { -inline uint64_t UnpackUint64Value_Le(const void* data) +inline uint64_t UnpackUint64ValueLe(const void* data) { return le64toh(*reinterpret_cast(data)); } -inline uint64_t UnpackUint64Value_Be(const void* data) +inline uint64_t UnpackUint64ValueBe(const void* data) { return be64toh(*reinterpret_cast(data)); } @@ -38,11 +38,11 @@ std::string U64ToHexString(uint64_t v); class BFloat16 { public: - static constexpr uint16_t value_mask = 0x7fff; - static constexpr uint16_t inf_value = 0x7f80; - static constexpr uint16_t nan_value = 0x7fc0; - static constexpr uint16_t true_value = 0x3c00; - static constexpr uint32_t f32_inf_value = 0x7f800000; + static constexpr uint16_t VALUE_MASK = 0x7fff; + static constexpr uint16_t INF_VALUE = 0x7f80; + static constexpr uint16_t NAN_VALUE = 0x7fc0; + static constexpr uint16_t TRUE_VALUE = 0x3c00; + static constexpr uint32_t F32_INF_VALUE = 0x7f800000; BFloat16() = default; ~BFloat16() = default; @@ -51,7 +51,7 @@ public: BFloat16 &operator=(const BFloat16 &other) noexcept = default; BFloat16 &operator=(BFloat16 &&other) noexcept = default; - explicit BFloat16(float f); + explicit BFloat16(float f32); explicit operator float() const; BFloat16 operator+(const BFloat16& other) const { return BFloat16(static_cast(*this) + static_cast(other)); } @@ -131,7 +131,7 @@ enum TensorFormat : int { FORMAT_HASHTABLE_LOOKUP_VALUE = 22, FORMAT_HASHTABLE_LOOKUP_OUTPUT = 23, FORMAT_HASHTABLE_LOOKUP_HITS = 24, - FORMAT_C1HWNCoC0 = 25, + FORMAT_C1HWNCOC0 = 25, FORMAT_MD = 26, FORMAT_NDHWC = 27, FORMAT_FRACTAL_ZZ = 28, diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.cpp b/debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.cpp index 7f025e568abdfe95830902d1e72bdb77300f7de5..d8861e5b0c766f1254063bdda568c6f9b2e21ef4 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.cpp @@ -18,9 +18,9 @@ #include #include -#include "FileUtils.hpp" -#include "DataUtils.hpp" -#include "FileOperation.hpp" +#include "FileUtils.h" +#include "DataUtils.h" +#include "FileOperation.h" namespace MindStudioDebugger { namespace FileOperation { @@ -34,7 +34,8 @@ struct NpyDtypeDescr { char type; size_t length; - std::string str() const { + std::string Str() const + { std::ostringstream buffer; buffer << "\'" << byteorder << type << length << "\'"; return buffer.str(); @@ -42,9 +43,9 @@ struct NpyDtypeDescr { }; // npy file header start information -constexpr char kNpyMagicPrefix[] = "\x93NUMPY"; -constexpr size_t kNpyMagicLen = sizeof(kNpyMagicPrefix) - 1; -constexpr size_t kNpyArrayAlign = 64; +constexpr char NPY_MAGIC_PREFIX[] = "\x93NUMPY"; +constexpr size_t NPY_MAGIC_LEN = sizeof(NPY_MAGIC_PREFIX) - 1; +constexpr size_t NPY_ARRAY_ALIGN = 64; static const std::unordered_map npyTypeDescMap = { {DataType::DT_BOOL, NpyDtypeDescr{'|', 'b', 1}}, {DataType::DT_INT8, NpyDtypeDescr{'|', 'i', 1}}, {DataType::DT_INT16, NpyDtypeDescr{'<', 'i', 2}}, {DataType::DT_INT32, NpyDtypeDescr{'<', 'i', 4}}, @@ -90,7 +91,8 @@ inline static std::string NpyTransShapeToStr(const DataUtils::TensorShape &shape return buffer.str(); } -inline static std::vector NpyLen2Bytes(size_t length, size_t lengthLen) { +inline static std::vector NpyLen2Bytes(size_t length, size_t lengthLen) +{ std::vector buff; lengthLen = std::min(lengthLen, static_cast(sizeof(length))); for (size_t i = 0; i < lengthLen; i++) { @@ -100,7 +102,8 @@ inline static std::vector NpyLen2Bytes(size_t length, size_t lengthLen) { return buff; } -static std::string GenerateNpyHeader(const DataUtils::TensorShape &shape, DataUtils::DataType dt, bool fortranOrder=false) +static std::string GenerateNpyHeader(const DataUtils::TensorShape &shape, + DataUtils::DataType dt, bool fortranOrder = false) { auto typeDesc = npyTypeDescMap.find(dt); if (typeDesc == npyTypeDescMap.end()) { @@ -111,7 +114,7 @@ static std::string GenerateNpyHeader(const DataUtils::TensorShape &shape, DataUt std::string fortranOrderStr = fortranOrder ? "True" : "False" ; buffer << "{"; - buffer << "'descr': " << typeDesc->second.str() << ", "; + buffer << "'descr': " << typeDesc->second.Str() << ", "; buffer << "'fortran_order': " << fortranOrderStr << ", "; buffer << "'shape': " << NpyTransShapeToStr(shape) << ", "; buffer << "}"; @@ -125,19 +128,19 @@ static std::string GenerateNpyHeader(const DataUtils::TensorShape &shape, DataUt constexpr const size_t lengthLenV2 = 4; size_t lengthLen = lengthLenV1; - size_t totalLen = kNpyMagicLen + versionLen + lengthLen + headerLen + 1; + size_t totalLen = NPY_MAGIC_LEN + versionLen + lengthLen + headerLen + 1; if (totalLen > maxLen) { version = {2, 0}; lengthLen = lengthLenV2; - totalLen = kNpyMagicLen + versionLen + lengthLen + headerLen + 1; + totalLen = NPY_MAGIC_LEN + versionLen + lengthLen + headerLen + 1; } - const size_t padLen = kNpyArrayAlign - totalLen % kNpyArrayAlign; + const size_t padLen = NPY_ARRAY_ALIGN - totalLen % NPY_ARRAY_ALIGN; const size_t paddingHeaderLen = headerLen + padLen + 1; const std::string padding(padLen, ' '); std::vector lengthBytes = NpyLen2Bytes(paddingHeaderLen, lengthLen); std::ostringstream out; - out.write(kNpyMagicPrefix, DataUtils::SizeToS64(kNpyMagicLen)); + out.write(NPY_MAGIC_PREFIX, DataUtils::SizeToS64(NPY_MAGIC_LEN)); out.put(version.first); out.put(version.second); out.write(lengthBytes.data(), DataUtils::SizeToS64(lengthBytes.size())); diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.hpp b/debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.h similarity index 95% rename from debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.hpp rename to debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.h index 3f89263ae3621d33f5bbc8a67e86887d8063067e..1560a1a6dba353f2e0122a639e46fa4c87195bba 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/FileOperation.h @@ -18,8 +18,8 @@ #include -#include "include/ErrorCode.hpp" -#include "DataUtils.hpp" +#include "include/ErrorCode.h" +#include "DataUtils.h" namespace MindStudioDebugger { diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.cpp b/debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.cpp index 246f899690ccd0e306f5b6b550870406086430cc..fddd4e28721c1cb9c0a8e61503743ced27022012 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.cpp @@ -27,8 +27,8 @@ #include #include -#include "include/ErrorCode.hpp" -#include "FileUtils.hpp" +#include "include/ErrorCode.h" +#include "FileUtils.h" /* 部分环境上c++版本比较老，这里不用filesystem库实现 */ @@ -38,7 +38,8 @@ namespace FileUtils { using namespace MindStudioDebugger; /********************* 基础检查函数库，不做过多校验，路径有效性由调用者保证 ******************/ -bool IsPathExist(const std::string& path) { +bool IsPathExist(const std::string& path) +{ struct stat buffer; return (stat(path.c_str(), &buffer) == 0); } @@ -60,7 +61,7 @@ static std::string GetFullPath(const std::string &originPath) } cwd = cwdBuf; - std::string fullPath = std::move(cwd + pathSeparator + originPath); + std::string fullPath = std::move(cwd + PATH_SEPARATOR + originPath); return fullPath; } @@ -84,7 +85,8 @@ std::vector SplitPath(const std::string &path, char separator) return tokens; } -std::string GetAbsPath(const std::string &originPath) { +std::string GetAbsPath(const std::string &originPath) +{ std::string fullPath = GetFullPath(originPath); if (fullPath.empty()) { return ""; @@ -118,7 +120,8 @@ std::string GetAbsPath(const std::string &originPath) { return resolvedPath; } -bool IsDir(const std::string& path) { +bool IsDir(const std::string& path) +{ struct stat buffer; if (stat(path.c_str(), &buffer) == 0) { return (buffer.st_mode & S_IFDIR) != 0; @@ -126,15 +129,17 @@ bool IsDir(const std::string& path) { return false; } -bool IsRegularFile(const std::string& path) { - struct stat path_stat; - if (stat(path.c_str(), &path_stat) == 0) { - return S_ISREG(path_stat.st_mode); +bool IsRegularFile(const std::string& path) +{ + struct stat pathStat; + if (stat(path.c_str(), &pathStat) == 0) { + return S_ISREG(pathStat.st_mode); } return false; } -bool IsFileSymbolLink(const std::string& path) { +bool IsFileSymbolLink(const std::string& path) +{ struct stat buffer; if (lstat(path.c_str(), &buffer) == 0) { if (S_ISLNK(buffer.st_mode)) { @@ -144,7 +149,8 @@ bool IsFileSymbolLink(const std::string& path) { return false; } -bool IsPathCharactersValid(const std::string& path) { +bool IsPathCharactersValid(const std::string& path) +{ for (const char& ch : path) { if (!std::isalnum(ch) && ch != '_' && ch != '.' && ch != ':' && ch != '/' && ch != '-') { return false; @@ -243,14 +249,14 @@ bool IsPathLengthLegal(const std::string& path) bool IsPathDepthValid(const std::string& path) { - return std::count(path.begin(), path.end(), pathSeparator) <= PATH_DEPTH_MAX; + return std::count(path.begin(), path.end(), PATH_SEPARATOR) <= PATH_DEPTH_MAX; } bool IsFileOwner(const std::string& path) { - struct stat file_stat; - if (stat(path.c_str(), &file_stat) == 0) { - if (file_stat.st_uid == getuid()) { + struct stat fileStat; + if (stat(path.c_str(), &fileStat) == 0) { + if (fileStat.st_uid == getuid()) { return true; } } @@ -306,7 +312,6 @@ static DebuggerErrno DeleteDirRec(const std::string &path, uint32_t depth) closedir(dir); return DebuggerErrno::ERROR_ILLEGAL_FILE_TYPE; } - } closedir(dir); @@ -321,7 +326,8 @@ static DebuggerErrno DeleteDirRec(const std::string &path, uint32_t depth) return DebuggerErrno::OK; } -DebuggerErrno DeleteDir(const std::string &path, bool recursion) { +DebuggerErrno DeleteDir(const std::string &path, bool recursion) +{ if (!IsPathExist(path)) { return DebuggerErrno::OK; } @@ -340,7 +346,8 @@ DebuggerErrno DeleteDir(const std::string &path, bool recursion) { return DebuggerErrno::OK; } -static DebuggerErrno CreateDirAux(const std::string& path, bool recursion, mode_t mode) { +static DebuggerErrno CreateDirAux(const std::string& path, bool recursion, mode_t mode) +{ std::string parent = GetParentDir(path); DebuggerErrno ret; @@ -404,16 +411,17 @@ DebuggerErrno Chmod(const std::string& path, const mode_t& mode) return chmod(absPath.c_str(), mode) == 0 ? DebuggerErrno::OK : DebuggerErrno::ERROR_SYSCALL_FAILED; } -DebuggerErrno GetFileSize(const std::string &path, size_t& size) { - struct stat path_stat; - if (stat(path.c_str(), &path_stat) != 0) { +DebuggerErrno GetFileSize(const std::string &path, size_t& size) +{ + struct stat pathStat; + if (stat(path.c_str(), &pathStat) != 0) { return DebuggerErrno::ERROR_FILE_NOT_EXISTS; } - if (!S_ISREG(path_stat.st_mode)) { + if (!S_ISREG(pathStat.st_mode)) { return DebuggerErrno::ERROR_ILLEGAL_FILE_TYPE; } - size = static_cast(path_stat.st_size); + size = static_cast(pathStat.st_size); return DebuggerErrno::OK; } @@ -600,63 +608,5 @@ DebuggerErrno CheckFileBeforeCreateOrWrite(const std::string &path, bool overwri } return DebuggerErrno::OK; } - -/* 其他文件操作工具 */ -static DebuggerErrno ListAllAux(const std::string &path, std::vector& output, uint32_t depth) -{ - if (depth > PATH_DEPTH_MAX) { - return DebuggerErrno::ERROR_PATH_TOO_DEEP; - } - - DIR* dir = opendir(path.c_str()); - if (dir == nullptr) { - return DebuggerErrno::ERROR_FAILED_TO_OPEN_FILE; - } - - DebuggerErrno ret = DebuggerErrno::OK; - size_t max = output.capacity(); - size_t num = output.size(); - if (num >= max) { - return DebuggerErrno::OK; - } - - struct dirent* entry = nullptr; - while ((entry = readdir(dir)) != nullptr) { - if (strcmp(entry->d_name, ".") == 0 || (strcmp(entry->d_name, "..") == 0)) { - continue; - } - std::string entryPath = path + "/" + entry->d_name; - if (entry->d_type == DT_DIR) { - ret = ListAllAux(entryPath, output, depth + 1); - if (ret != DebuggerErrno::OK) { - closedir(dir); - return ret; - } - } else if (entry->d_type == DT_REG) { - output.emplace_back(entryPath); - if (++num >= max) { - break; - } - } - } - closedir(dir); - return DebuggerErrno::OK; -} - -std::vector ListAll(const std::string &path, size_t max) -{ - std::vector ret; - std::string realPath = GetAbsPath(path); - if (CheckDirCommon(realPath) != DebuggerErrno::OK) { - return ret; - } - ret.reserve(max); - - uint32_t depth = std::count(realPath.begin(), realPath.end(), pathSeparator); - ListAllAux(realPath, ret, depth); - ret.resize(ret.size()); - return ret; -} - } } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.hpp b/debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.h similarity index 83% rename from debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.hpp rename to debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.h index 70b47137fc40fd7fb73be11ddb8d3551550e2b8d..e3814ad7cdf2a6f41e9193849f10c6a2fd6e0d92 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/FileUtils.h @@ -23,11 +23,11 @@ #include #include -#include "include/ErrorCode.hpp" +#include "include/ErrorCode.h" namespace MindStudioDebugger { -constexpr const char pathSeparator = '/'; +constexpr const char PATH_SEPARATOR = '/'; constexpr const uint32_t FULL_PATH_LENGTH_MAX = 4096; constexpr const uint32_t FILE_NAME_LENGTH_MAX = 255; constexpr const uint32_t PATH_DEPTH_MAX = 32; @@ -64,8 +64,8 @@ constexpr const uint32_t FILE_NAME_MAX = 255; /* 基础检查函数库，不做过多校验，路径有效性由调用者保证 */ bool IsPathExist(const std::string& path); -std::vector SplitPath(const std::string &path, char separator=pathSeparator); -std::string GetAbsPath(const std::string &path); +std::vector SplitPath(const std::string &path, char separator = PATH_SEPARATOR); +std::string GetAbsPath(const std::string &originpath); bool IsDir(const std::string& path); bool IsRegularFile(const std::string& path); bool IsFileSymbolLink(const std::string& path); @@ -85,23 +85,19 @@ bool IsFileOwner(const std::string& path); /* 文件操作函数库，会对入参做基本检查 */ DebuggerErrno DeleteFile(const std::string &path); -DebuggerErrno DeleteDir(const std::string &path, bool recursion=false); -DebuggerErrno CreateDir(const std::string &path, bool recursion=false, mode_t mode=NORMAL_DIR_MODE_DEFAULT); +DebuggerErrno DeleteDir(const std::string &path, bool recursion = false); +DebuggerErrno CreateDir(const std::string &path, bool recursion = false, mode_t mode = NORMAL_DIR_MODE_DEFAULT); DebuggerErrno Chmod(const std::string& path, const mode_t& mode); DebuggerErrno GetFileSize(const std::string &path, size_t& size); -DebuggerErrno OpenFile(const std::string& path, std::ifstream& ifs, std::ios::openmode mode=std::ios::in); -DebuggerErrno OpenFile(const std::string& path, std::ofstream& ofs, std::ios::openmode mode=std::ios::out, - mode_t permission=NORMAL_FILE_MODE_DEFAULT); +DebuggerErrno OpenFile(const std::string& path, std::ifstream& ifs, std::ios::openmode mode = std::ios::in); +DebuggerErrno OpenFile(const std::string& path, std::ofstream& ofs, std::ios::openmode mode = std::ios::out, + mode_t permission = NORMAL_FILE_MODE_DEFAULT); /* 通用检查函数 */ DebuggerErrno CheckFileSuffixAndSize(const std::string &path, FileType type); DebuggerErrno CheckDirCommon(const std::string &path); DebuggerErrno CheckFileBeforeRead(const std::string &path, const std::string& authority="r", - FileType type=FileType::COMMON); -DebuggerErrno CheckFileBeforeCreateOrWrite(const std::string &path, bool overwrite=false); - -/* 其他文件操作工具 */ -std::vector ListAll(const std::string &path, size_t max = 1024); - + FileType type = FileType::COMMON); +DebuggerErrno CheckFileBeforeCreateOrWrite(const std::string &path, bool overwrite = false); } } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.cpp b/debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.cpp index 27111d60c9f86f2ae9b2b2a00b804ab886917755..1c1a4e96965e014f038d85636627c7b2ec185814 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.cpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.cpp @@ -68,13 +68,13 @@ std::string CalculateMD5(const uint8_t* data, size_t length) unsigned char digest[MD5_DIGEST_LENGTH]; MD5_Final(digest, &md5ctx); - static const char hexchar[] = "0123456789abcdef"; + static const char HEX_CHAR[] = "0123456789abcdef"; constexpr const uint8_t hexbase = 16; constexpr const size_t byteToStrWidth = 2; char md5string[MD5_DIGEST_LENGTH * byteToStrWidth + 1]; for (int i = 0; i < MD5_DIGEST_LENGTH; i++) { - md5string[i * byteToStrWidth] = hexchar[digest[i] / hexbase]; - md5string[i * byteToStrWidth + 1] = hexchar[digest[i] % hexbase]; + md5string[i * byteToStrWidth] = HEX_CHAR[digest[i] / hexbase]; + md5string[i * byteToStrWidth + 1] = HEX_CHAR[digest[i] % hexbase]; } md5string[sizeof(md5string) - 1] = '\0'; diff --git a/debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.hpp b/debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.h similarity index 88% rename from debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.hpp rename to debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.h index 141471ac8ce284ac1a7ab4b6db59f5d0da9a9fe2..accbee3187f02ba81e2eaf4e550a36200f52ec58 100644 --- a/debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.hpp +++ b/debug/accuracy_tools/msprobe/ccsrc/utils/MathUtils.h @@ -23,7 +23,8 @@ namespace MindStudioDebugger { namespace MathUtils { template -T Gcd(T a, T b) { +T Gcd(T a, T b) +{ if (a == 0 || b == 0) { return 0; } @@ -37,7 +38,8 @@ T Gcd(T a, T b) { } template -T Lcm(T a, T b) { +T Lcm(T a, T b) +{ if (a == 0 || b == 0) { return 0; } @@ -46,7 +48,8 @@ T Lcm(T a, T b) { } template -T DivCeil(T v, T divisor) { +T DivCeil(T v, T divisor) +{ if (divisor == 0) { return 0; } @@ -56,13 +59,13 @@ T DivCeil(T v, T divisor) { template T AlignCeil(T v, T block) { - return DivCeil(v, block) * block; + return DivCeil(v, block) * block; } float Random(); float Random(float floor, float ceil); int32_t RandomInt(int32_t floor, int32_t ceil); -std::string RandomString(uint32_t len, char min=' ', char max='~'); +std::string RandomString(uint32_t len, char min = ' ', char max = '~'); std::string CalculateMD5(const uint8_t* data, size_t length); diff --git a/debug/accuracy_tools/msprobe/core/__init__.py b/debug/accuracy_tools/msprobe/core/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..eb80022b66670467408474bec6f5f46e48ff29b2 100644 --- a/debug/accuracy_tools/msprobe/core/__init__.py +++ b/debug/accuracy_tools/msprobe/core/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from msprobe.core.single_save.single_saver import SingleSave +from msprobe.core.single_save.single_comparator import SingleComparator diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index f7ba9f90d0e27df9a40ee37b838fbd2919f0d25b..c30440f156afde9f4db483813a22fea1123b8627 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -51,7 +51,10 @@ class Const: FOUR_SEGMENT = 4 SIX_SEGMENT = 6 SEVEN_SEGMENT = 7 + MAX_DEPTH = 10 + CPU_QUARTER = 4 + DUMP_MAX_DEPTH = 50 # dump mode ALL = "all" @@ -67,7 +70,7 @@ class Const: SUMMARY = "summary" MD5 = "md5" VALUE = "value" - SUMMARY_MODE = [ALL, SUMMARY, MD5] + SUMMARY_MODE = ["statistics", "md5"] WRITE_FLAGS = os.O_WRONLY | os.O_CREAT WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR @@ -77,6 +80,8 @@ class Const: NUMPY_SUFFIX = ".npy" NUMPY_PATTERN = "*.npy" PT_SUFFIX = ".pt" + PY_SUFFIX = ".py" + INIT_PY = "init.py" ONE_GB = 1073741824 # 1 * 1024 * 1024 * 1024 TEN_GB = 10737418240 # 10 * 1024 * 1024 * 1024 ONE_MB = 1048576 # 1 * 1024 * 1024 @@ -92,6 +97,7 @@ class Const: GRAD_OUTPUT = 'grad_output' PARAMS = 'parameters' PARAMS_GRAD = 'parameters_grad' + DEBUG = 'debug' START = "start" STOP = "stop" ENV_ENABLE = "1" @@ -129,6 +135,7 @@ class Const: NPU = 'NPU' NPU_LOWERCASE = 'npu' CPU_LOWERCASE = 'cpu' + GPU_LOWERCASE = 'gpu' CUDA_LOWERCASE = 'cuda' DEVICE = 'device' DISTRIBUTED = 'Distributed' @@ -137,6 +144,10 @@ class Const: MODULE_PREFIX = ["Module", "Cell"] FORWARD_NAME_SUFFIX = ".forward" + DUMP_JSON_FILE = "dump_json_file" + DEBUG_JSON_FILE = "debug_json_file" + STACK_JSON_FILE = "stack_json_file" + # struct json param ORIGIN_DATA = "origin_data" SCOPE = "scope" @@ -188,7 +199,11 @@ class Const: FILL_CHAR_NUMS = 50 TOOL_ENDS_SUCCESSFULLY = f"{TOOL_NAME} ends successfully." + WITHOUT_CALL_STACK = "The call stack retrieval failed." + STACK_FILTER_KEYWORDS = ["msprobe/core", "msprobe/pytorch", "msprobe/mindspore"] + CALL_STACK_FLAG = "data_dump/api_registry" + NEW_STACK_FLAG = "0" STEP = "step" RANK = "rank" @@ -206,12 +221,16 @@ class Const: TORCH_FLOAT32 = "torch.float32" TORCH_BFLOAT16 = "torch.bfloat16" + TYPE = 'type' DTYPE = 'dtype' SHAPE = 'shape' + STACK_INFO = 'stack_info' MAX = 'Max' MIN = 'Min' MEAN = 'Mean' NORM = 'Norm' + DATA_NAME = 'data_name' + TENSOR_STAT_INDEX = 'tensor_stat_index' CODE_STACK = 'Code Stack' OP_NAME = 'Op Name' @@ -224,6 +243,9 @@ class Const: SCOPE_SEPARATOR = "/" REPLACEMENT_CHARACTER = "_" + FORWARD_PATTERN = SEP + FORWARD + SEP + BACKWARD_PATTERN = SEP + BACKWARD + SEP + OPTIMIZER = "optimizer" CLIP_GRAD = "clip_grad" END_PREFIX = "end_" @@ -240,6 +262,7 @@ class Const: PT_API_TYPE_ATEN = "aten" PT_API_TYPE_DIST = "distributed" PT_API_TYPE_NPU_DIST = "npu_distributed" + PT_API_TYPE_MINDSPEED = "mindspeed" MS_API_TYPE_OPS = "ops" MS_API_TYPE_TENSOR = "tensor" @@ -247,6 +270,7 @@ class Const: MS_API_TYPE_MINT = "mint.ops" MS_API_TYPE_MINT_FUNC = "mint.nn.functional" MS_API_TYPE_COM = "communication.comm_func" + MS_API_TYPE_MINT_DIST = "mint.distributed" FUNCTIONAL_API_TYPE_PREFIX = "Functional" TENSOR_API_TYPE_PREFIX = "Tensor" @@ -256,9 +280,11 @@ class Const: NPU_API_TYPE_PREFIX = "NPU" ATEN_API_TYPE_PREFIX = "Aten" VF_API_TYPE_PREFIX = "VF" + MINDSPEED_API_TYPE_PREFIX = "MindSpeed" MINT_API_TYPE_PREFIX = "Mint" MINT_FUNC_API_TYPE_PREFIX = "MintFunctional" + MINT_DIST_API_TYPE_PREFIX = "MintDistributed" SUPPORT_API_DICT_KEY_MAP = { PT_FRAMEWORK: { @@ -269,7 +295,8 @@ class Const: PT_API_TYPE_NPU: PT_API_TYPE_NPU, PT_API_TYPE_ATEN: PT_API_TYPE_ATEN, PT_API_TYPE_DIST: PT_API_TYPE_DIST, - PT_API_TYPE_NPU_DIST: PT_API_TYPE_NPU_DIST + PT_API_TYPE_NPU_DIST: PT_API_TYPE_NPU_DIST, + PT_API_TYPE_MINDSPEED: PT_API_TYPE_MINDSPEED }, MS_FRAMEWORK: { MS_API_TYPE_OPS: MS_API_TYPE_OPS, @@ -277,7 +304,8 @@ class Const: MS_API_TYPE_STUB_TENSOR: MS_API_TYPE_TENSOR, MS_API_TYPE_MINT: MS_API_TYPE_MINT, MS_API_TYPE_MINT_FUNC: MS_API_TYPE_MINT_FUNC, - MS_API_TYPE_COM: MS_API_TYPE_COM + MS_API_TYPE_COM: MS_API_TYPE_COM, + MS_API_TYPE_MINT_DIST: MS_API_TYPE_MINT_DIST }, MT_FRAMEWORK: { PT_API_TYPE_FUNCTIONAL: PT_API_TYPE_FUNCTIONAL, @@ -297,7 +325,8 @@ class Const: PT_API_TYPE_NPU: NPU_API_TYPE_PREFIX, PT_API_TYPE_ATEN: ATEN_API_TYPE_PREFIX, PT_API_TYPE_DIST: DIST_API_TYPE_PREFIX, - PT_API_TYPE_NPU_DIST: DIST_API_TYPE_PREFIX + PT_API_TYPE_NPU_DIST: DIST_API_TYPE_PREFIX, + PT_API_TYPE_MINDSPEED: MINDSPEED_API_TYPE_PREFIX }, MS_FRAMEWORK: { MS_API_TYPE_OPS: FUNCTIONAL_API_TYPE_PREFIX, @@ -305,7 +334,8 @@ class Const: MS_API_TYPE_STUB_TENSOR: TENSOR_API_TYPE_PREFIX, MS_API_TYPE_MINT: MINT_API_TYPE_PREFIX, MS_API_TYPE_MINT_FUNC: MINT_FUNC_API_TYPE_PREFIX, - MS_API_TYPE_COM: DIST_API_TYPE_PREFIX + MS_API_TYPE_COM: DIST_API_TYPE_PREFIX, + MS_API_TYPE_MINT_DIST: MINT_DIST_API_TYPE_PREFIX }, MT_FRAMEWORK: { PT_API_TYPE_FUNCTIONAL: FUNCTIONAL_API_TYPE_PREFIX, @@ -316,6 +346,35 @@ class Const: } } + def _fused_adamw_( + self, + grads, + exp_avgs, + exp_avg_sqs, + max_exp_avg_sqs, + state_steps, + *, + lr, + beta1, + beta2, + weight_decay, + eps, + amsgrad, + maximize, + grad_scale=None, + found_inf=None + ): + pass + + API_WITH_SELF_ARG = { + 'Torch._fused_adamw_': _fused_adamw_ + } + + ASCEND = "ASCEND" + MATCH_MODE_NAME = "pure name" + MATCH_MODE_MAPPING = "mapping" + MATCH_MODE_SIMILARITY = "similarity" + class CompareConst: """ @@ -365,6 +424,7 @@ class CompareConst: OUTPUT_STRUCT = "output_struct" PARAMS_STRUCT = "params_struct" PARAMS_GRAD_STRUCT = "params_grad_struct" + DEBUG_STRUCT = "debug_struct" SUMMARY = "summary" COMPARE_RESULT = "compare_result" COMPARE_MESSAGE = "compare_message" @@ -471,16 +531,10 @@ class CompareConst: Const.KWARGS: INPUT_STRUCT, Const.OUTPUT: OUTPUT_STRUCT, Const.PARAMS: PARAMS_STRUCT, - Const.PARAMS_GRAD: PARAMS_GRAD_STRUCT + Const.PARAMS_GRAD: PARAMS_GRAD_STRUCT, + Const.DEBUG: DEBUG_STRUCT } - STRUCT_COMPARE_KEY = [ - INPUT_STRUCT, - OUTPUT_STRUCT, - PARAMS_STRUCT, - PARAMS_GRAD_STRUCT - ] - # compare standard HUNDRED_RATIO_THRESHOLD = 0.01 THOUSAND_RATIO_THRESHOLD = 0.001 @@ -559,15 +613,35 @@ class CompareConst: MAX_DIFF: None, MIN_DIFF: None, MEAN_DIFF: None, NORM_DIFF: None, MAX_RELATIVE_ERR: None, MIN_RELATIVE_ERR: None, MEAN_RELATIVE_ERR: None, NORM_RELATIVE_ERR: None } + + API_MAPPING_KEYS_TO_COMPARE = [ + ('ms_args', 'pt_args'), + ('ms_outputs', 'pt_outputs'), + ('ms_parameters', 'pt_parameters'), + ('ms_parameters_grad', 'pt_parameters_grad') + ] + INPUT_PATTERN = Const.SEP + Const.INPUT + Const.SEP KWARGS_PATTERN = Const.SEP + Const.KWARGS + Const.SEP OUTPUT_PATTERN = Const.SEP + Const.OUTPUT + Const.SEP PARAMS_PATTERN = Const.SEP + Const.PARAMS + Const.SEP PARAMS_GRAD_PATTERN = Const.SEP + Const.PARAMS_GRAD + Const.SEP - COMPARE_KEY = 'compare_key' - COMPARE_SHAPE = 'compare_shape' + + CMP_KEY = 'compare_key' + CMP_SHAPE = 'compare_shape' + + OP_NAME_X = 'op_name_x' + MATCH_RESULT_COLUMNS = [ + OP_NAME_X, 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + CMP_KEY, CMP_SHAPE, + 'op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y', + ] + INTERNAL_API_MAPPING_FILE = 'ms_to_pt_api.yaml' UNREADABLE = 'unreadable data' + NPU_DUMP_DATA_DIR = 'npu_dump_data_dir' + BENCH_DUMP_DATA_DIR = 'bench_dump_data_dir' + NO_REAL_DATA_FLAG = '-1' class FileCheckConst: @@ -589,6 +663,8 @@ class FileCheckConst: XLSX_SUFFIX = ".xlsx" YAML_SUFFIX = ".yaml" IR_SUFFIX = ".ir" + ZIP_SUFFIX = ".zip" + SHELL_SUFFIX = ".sh" MAX_PKL_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 MAX_NUMPY_SIZE = 10737418240 # 10 * 1024 * 1024 * 1024 MAX_JSON_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 @@ -597,6 +673,8 @@ class FileCheckConst: MAX_XLSX_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 MAX_YAML_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 MAX_IR_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 + MAX_ZIP_SIZE = 10737418240 # 10 * 1024 * 1024 * 1024 + MAX_FILE_IN_ZIP_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 COMMOM_FILE_SIZE = 1048576 # 1 * 1024 * 1024 DIR = "dir" FILE = "file" @@ -610,7 +688,8 @@ class FileCheckConst: CSV_SUFFIX: MAX_CSV_SIZE, XLSX_SUFFIX: MAX_XLSX_SIZE, YAML_SUFFIX: MAX_YAML_SIZE, - IR_SUFFIX: MAX_IR_SIZE + IR_SUFFIX: MAX_IR_SIZE, + ZIP_SUFFIX: MAX_ZIP_SIZE } CSV_BLACK_LIST = r'^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]' @@ -659,6 +738,15 @@ class MonitorConst: """ Class for monitor const """ + + # monitor config set default values + DEFAULT_GRAD_ACC_STEPS = 1 + DEFAULT_START_ITERATION = 0 + DEFAULT_START_STEP = 0 + DEFAULT_MAX_COLLECT_TIMES = 1e8 + DEFAULT_MIN_COLLECT_TIMES = 0 + DEFAULT_STEP_INTERVAL = 1 + OP_LIST = ["norm", "min", "max", "zeros", "nans", "id", "mean"] MONITOR_OUTPUT_DIR = "MONITOR_OUTPUT_DIR" DEFAULT_MONITOR_OUTPUT_DIR = "./monitor_output" @@ -671,7 +759,7 @@ class MonitorConst: "DeepSpeedZeroOptimizer_Stage3" ) DEEPSPEED_ZERO_OPT_FILTER = "DeepSpeedZeroOptimizer" - RULE_NAME = ['AnomalyTurbulence'] + RULE_NAME = ['AnomalyTurbulence', 'AnomalyNan'] SLICE_SIZE = 20480 # used for name @@ -688,12 +776,13 @@ class MonitorConst: ACTVGRAD = "actv_grad" POST_GRAD = "post_grad" PRE_GRAD = "pre_grad" + PRE_PARAM = "param_origin" + POST_PARAM = "param_updated" ACC_GRAD = "acc_grad" PREFIX_POST = "post" PREFIX_PRE = "pre" EXP_AVG = "exp_avg" EXP_AVG_SQ = "exp_avg_sq" - PARAM = "param" CSV_HEADER = ["vpp_stage", "name", "step"] CSV_HEADER_XY = ["vpp_stage", "name", "step", "micro_step"] @@ -704,3 +793,5 @@ class MonitorConst: CSV = "csv" API = "api" HEADER_NAME = 'name' + + MAX_NDIGITS = 20 diff --git a/debug/accuracy_tools/msprobe/core/common/decorator.py b/debug/accuracy_tools/msprobe/core/common/decorator.py new file mode 100644 index 0000000000000000000000000000000000000000..d3710002bcc281be2fd0f19fc7abda1af35ec936 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/common/decorator.py @@ -0,0 +1,50 @@ +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict +from functools import wraps + +from msprobe.core.common.const import Const +from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common.log import logger + +# 记录工具函数递归的深度 +recursion_depth = defaultdict(int) + + +def recursion_depth_decorator(func_info, max_depth=Const.MAX_DEPTH): + """装饰一个函数，当函数递归调用超过限制时，抛出异常并打印函数信息。""" + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + func_id = id(func) + recursion_depth[func_id] += 1 + if recursion_depth[func_id] > max_depth: + msg = f"call {func_info} exceeds the recursion limit." + logger.error_log_with_exp( + msg, + MsprobeException( + MsprobeException.RECURSION_LIMIT_ERROR, msg + ), + ) + try: + result = func(*args, **kwargs) + finally: + recursion_depth[func_id] -= 1 + return result + + return wrapper + + return decorator diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index fdc626ca6a1a90e9060cefa237f9d5d8d7e42844..b967092f977a5e528bfaea7cf0a8c6cdab3282a1 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -12,23 +12,33 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import atexit import csv import fcntl +import io import os +import pickle +from multiprocessing import shared_memory import stat import json import re import shutil +import sys +import zipfile +import multiprocessing from datetime import datetime, timezone from dateutil import parser import yaml import numpy as np import pandas as pd +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.global_lock import global_lock, is_main_process + +proc_lock = multiprocessing.Lock() class FileChecker: @@ -164,6 +174,12 @@ def check_path_exists(path): if not os.path.exists(path): logger.error('The file path %s does not exist.' % path) raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) + + +def check_path_not_exists(path): + if os.path.exists(path): + logger.error('The file path %s already exist.' % path) + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) def check_path_readability(path): @@ -266,6 +282,7 @@ def make_dir(dir_path): file_check.common_check() +@recursion_depth_decorator('msprobe.core.common.file_utils.create_directory', max_depth=16) def create_directory(dir_path): """ Function Description: @@ -297,12 +314,13 @@ def check_path_before_create(path): def check_dirpath_before_read(path): path = os.path.realpath(path) dirpath = os.path.dirname(path) - if check_others_writable(dirpath): - logger.warning(f"The directory is writable by others: {dirpath}.") - try: - check_path_owner_consistent(dirpath) - except FileCheckException: - logger.warning(f"The directory {dirpath} is not yours.") + if dedup_log('check_dirpath_before_read', dirpath): + if check_others_writable(dirpath): + logger.warning(f"The directory is writable by others: {dirpath}.") + try: + check_path_owner_consistent(dirpath) + except FileCheckException: + logger.warning(f"The directory {dirpath} is not yours.") def check_file_or_directory_path(path, isdir=False): @@ -332,6 +350,23 @@ def change_mode(path, mode): 'Failed to change {} authority. {}'.format(path, str(ex))) from ex +@recursion_depth_decorator('msprobe.core.common.file_utils.recursive_chmod') +def recursive_chmod(path): + """ + 递归地修改目录及其子目录和文件的权限，文件修改为640，路径修改为750 + + :param path: 要修改权限的目录路径 + """ + for _, dirs, files in os.walk(path): + for file_name in files: + file_path = os.path.join(path, file_name) + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + for dir_name in dirs: + dir_path = os.path.join(path, dir_name) + change_mode(dir_path, FileCheckConst.DATA_DIR_AUTHORITY) + recursive_chmod(dir_path) + + def path_len_exceeds_limit(file_path): return len(os.path.realpath(file_path)) > FileCheckConst.DIRECTORY_LENGTH or \ len(os.path.basename(file_path)) > FileCheckConst.FILE_NAME_LENGTH @@ -511,7 +546,7 @@ def write_csv(data, filepath, mode="a+", malicious_check=False): if not isinstance(value, str): return True try: - # -1.00 or +1.00 should be consdiered as digit numbers + # -1.00 or +1.00 should be considered as digit numbers float(value) except ValueError: # otherwise, they will be considered as formular injections @@ -557,7 +592,7 @@ def write_df_to_csv(data, filepath, mode="w", header=True, malicious_check=False if not isinstance(value, str): return True try: - # -1.00 or +1.00 should be consdiered as digit numbers + # -1.00 or +1.00 should be considered as digit numbers float(value) except ValueError: # otherwise, they will be considered as formular injections @@ -632,7 +667,7 @@ def os_walk_for_files(path, depth): return res -def check_crt_valid(pem_path): +def check_crt_valid(pem_path, is_public_key=False): """ Check the validity of the SSL certificate. @@ -641,6 +676,7 @@ def check_crt_valid(pem_path): Parameters: pem_path (str): The file path of the SSL certificate. + is_public_key (bool): The file is public key or not. Raises: RuntimeError: If the SSL certificate is invalid or expired. @@ -649,7 +685,10 @@ def check_crt_valid(pem_path): try: with FileOpen(pem_path, "r") as f: pem_data = f.read() - cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem_data) + if is_public_key: + cert = OpenSSL.crypto.load_publickey(OpenSSL.crypto.FILETYPE_PEM, pem_data) + else: + cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pem_data) pem_start = parser.parse(cert.get_notBefore().decode("UTF-8")) pem_end = parser.parse(cert.get_notAfter().decode("UTF-8")) logger.info(f"The SSL certificate passes the verification and the validity period " @@ -663,11 +702,229 @@ def check_crt_valid(pem_path): raise RuntimeError(f"The SSL certificate has expired and needs to be replaced, {pem_path}") -def read_xlsx(file_path): +def read_xlsx(file_path, sheet_name=None): check_file_or_directory_path(file_path) try: - result_df = pd.read_excel(file_path, keep_default_na=False) + if sheet_name: + result_df = pd.read_excel(file_path, keep_default_na=False, sheet_name=sheet_name) + else: + result_df = pd.read_excel(file_path, keep_default_na=False) except Exception as e: logger.error(f"The xlsx file failed to load. Please check the path: {file_path}.") raise RuntimeError(f"Read xlsx file {file_path} failed.") from e return result_df + + +def create_file_with_list(result_list, filepath): + check_path_before_create(filepath) + filepath = os.path.realpath(filepath) + try: + with FileOpen(filepath, 'w', encoding='utf-8') as file: + fcntl.flock(file, fcntl.LOCK_EX) + for item in result_list: + file.write(item + '\n') + fcntl.flock(file, fcntl.LOCK_UN) + except Exception as e: + logger.error(f'Save list to file "{os.path.basename(filepath)}" failed.') + raise RuntimeError(f"Save list to file {os.path.basename(filepath)} failed.") from e + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + +def create_file_with_content(data, filepath): + check_path_before_create(filepath) + filepath = os.path.realpath(filepath) + try: + with FileOpen(filepath, 'w', encoding='utf-8') as file: + fcntl.flock(file, fcntl.LOCK_EX) + file.write(data) + fcntl.flock(file, fcntl.LOCK_UN) + except Exception as e: + logger.error(f'Save content to file "{os.path.basename(filepath)}" failed.') + raise RuntimeError(f"Save content to file {os.path.basename(filepath)} failed.") from e + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + +def add_file_to_zip(zip_file_path, file_path, arc_path=None): + """ + Add a file to a ZIP archive, if zip does not exist, create one. + + :param zip_file_path: Path to the ZIP archive + :param file_path: Path to the file to add + :param arc_path: Optional path inside the ZIP archive where the file should be added + """ + check_file_suffix(zip_file_path, FileCheckConst.ZIP_SUFFIX) + check_file_size(file_path, FileCheckConst.MAX_FILE_IN_ZIP_SIZE) + zip_size = os.path.getsize(zip_file_path) if os.path.exists(zip_file_path) else 0 + if zip_size + os.path.getsize(file_path) > FileCheckConst.MAX_ZIP_SIZE: + raise RuntimeError(f"ZIP file size exceeds the limit of {FileCheckConst.MAX_ZIP_SIZE} bytes") + check_path_before_create(zip_file_path) + try: + proc_lock.acquire() + with zipfile.ZipFile(zip_file_path, 'a') as zip_file: + zip_file.write(file_path, arc_path) + except Exception as e: + logger.error(f'add file to zip "{os.path.basename(zip_file_path)}" failed.') + raise RuntimeError(f"add file to zip {os.path.basename(zip_file_path)} failed.") from e + finally: + proc_lock.release() + change_mode(zip_file_path, FileCheckConst.DATA_FILE_AUTHORITY) + + +def create_file_in_zip(zip_file_path, file_name, content): + """ + Create a file with content inside a ZIP archive. + + :param zip_file_path: Path to the ZIP archive + :param file_name: Name of the file to create + :param content: Content to write to the file + """ + check_file_suffix(zip_file_path, FileCheckConst.ZIP_SUFFIX) + check_path_before_create(zip_file_path) + zip_size = os.path.getsize(zip_file_path) if os.path.exists(zip_file_path) else 0 + if zip_size + sys.getsizeof(content) > FileCheckConst.MAX_ZIP_SIZE: + raise RuntimeError(f"ZIP file size exceeds the limit of {FileCheckConst.MAX_ZIP_SIZE} bytes") + try: + with open(zip_file_path, 'a+') as f: # 必须用 'a+' 模式才能 flock + # 2. 获取排他锁（阻塞直到成功） + fcntl.flock(f, fcntl.LOCK_EX) # LOCK_EX: 独占锁 + with zipfile.ZipFile(zip_file_path, 'a') as zip_file: + zip_info = zipfile.ZipInfo(file_name) + zip_info.compress_type = zipfile.ZIP_DEFLATED + zip_file.writestr(zip_info, content) + fcntl.flock(f, fcntl.LOCK_UN) + except Exception as e: + logger.error(f'Save content to file "{os.path.basename(zip_file_path)}" failed.') + raise RuntimeError(f"Save content to file {os.path.basename(zip_file_path)} failed.") from e + change_mode(zip_file_path, FileCheckConst.DATA_FILE_AUTHORITY) + + +def extract_zip(zip_file_path, extract_dir): + """ + Extract the contents of a ZIP archive to a specified directory. + + :param zip_file_path: Path to the ZIP archive + :param extract_dir: Directory to extract the contents to + """ + check_file_suffix(zip_file_path, FileCheckConst.ZIP_SUFFIX) + try: + proc_lock.acquire() + with zipfile.ZipFile(zip_file_path, 'r') as zip_file: + total_size = 0 + if len(zip_file.infolist()) > FileCheckConst.MAX_FILE_IN_ZIP_SIZE: + raise ValueError(f"Too many files in {os.path.basename(zip_file_path)}") + for file_info in zip_file.infolist(): + if file_info.file_size > FileCheckConst.MAX_FILE_IN_ZIP_SIZE: + raise ValueError(f"File {file_info.filename} is too large to extract") + + total_size += file_info.file_size + if total_size > FileCheckConst.MAX_ZIP_SIZE: + raise ValueError(f"Total extracted size exceeds the limit of {FileCheckConst.MAX_ZIP_SIZE} bytes") + except Exception as e: + logger.error(f'Save content to file "{os.path.basename(zip_file_path)}" failed.') + raise RuntimeError(f"Save content to file {os.path.basename(zip_file_path)} failed.") from e + finally: + proc_lock.release() + with zipfile.ZipFile(zip_file_path, 'r') as zip_file: + zip_file.extractall(extract_dir) + + +def split_zip_file_path(zip_file_path): + check_file_suffix(zip_file_path, FileCheckConst.ZIP_SUFFIX) + zip_file_path = os.path.realpath(zip_file_path) + return os.path.dirname(zip_file_path), os.path.basename(zip_file_path) + + +def dedup_log(func_name, filter_name): + with SharedDict() as shared_dict: + exist_names = shared_dict.get(func_name, set()) + if filter_name in exist_names: + return False + exist_names.add(filter_name) + shared_dict[func_name] = exist_names + return True + + +class SharedDict: + def __init__(self): + self._changed = False + self._dict = None + self._shm = None + + def __enter__(self): + self._load_shared_memory() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + try: + if self._changed: + data = pickle.dumps(self._dict) + global_lock.acquire() + self._shm.buf[0:len(data)] = bytearray(data) + global_lock.release() + self._shm.close() + except FileNotFoundError: + name = self.get_shared_memory_name() + logger.warning(f'close shared memory {name} failed, shared memory has already been destroyed.') + + def __setitem__(self, key, value): + self._dict[key] = value + self._changed = True + + def __contains__(self, item): + return item in self._dict + + @classmethod + def destroy_shared_memory(cls): + if is_main_process(): + name = cls.get_shared_memory_name() + try: + shm = shared_memory.SharedMemory(create=False, name=name) + shm.close() + shm.unlink() + logger.debug(f'destroy shared memory, name: {name}') + except FileNotFoundError: + logger.warning(f'destroy shared memory {name} failed, shared memory has already been destroyed.') + + @classmethod + def get_shared_memory_name(cls): + if is_main_process(): + return f'shared_memory_{os.getpid()}' + return f'shared_memory_{os.getppid()}' + + def get(self, key, default=None): + return self._dict.get(key, default) + + def _load_shared_memory(self): + name = self.get_shared_memory_name() + try: + self._shm = shared_memory.SharedMemory(create=False, name=name) + except FileNotFoundError: + try: + self._shm = shared_memory.SharedMemory(create=True, name=name, size=1024 * 1024) + data = pickle.dumps({}) + self._shm.buf[0:len(data)] = bytearray(data) + logger.debug(f'create shared memory, name: {name}') + except FileExistsError: + self._shm = shared_memory.SharedMemory(create=False, name=name) + self._safe_load() + + def _safe_load(self): + with io.BytesIO(self._shm.buf[:]) as buff: + try: + self._dict = SafeUnpickler(buff).load() + except Exception as e: + logger.warning(f'shared dict is unreadable, reason: {e}, create new dict.') + self._dict = {} + self._changed = True + + +class SafeUnpickler(pickle.Unpickler): + WHITELIST = {'builtins': {'str', 'bool', 'int', 'float', 'list', 'set', 'dict'}} + + def find_class(self, module, name): + if module in self.WHITELIST and name in self.WHITELIST[module]: + return super().find_class(module, name) + raise pickle.PicklingError(f'Unpickling {module}.{name} is illegal!') + + +atexit.register(SharedDict.destroy_shared_memory) diff --git a/debug/accuracy_tools/msprobe/core/common/framework_adapter.py b/debug/accuracy_tools/msprobe/core/common/framework_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..02ebbfc0a0639e32a9e89fca49d5ed13d8bc44f6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/common/framework_adapter.py @@ -0,0 +1,164 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.import functools +import functools +from msprobe.core.common.const import Const +from msprobe.core.common.file_utils import check_file_or_directory_path +from msprobe.core.common.file_utils import save_npy + + +class FrameworkDescriptor: + def __get__(self, instance, owner): + if owner._framework is None: + owner.import_framework() + return owner._framework + + +class FmkAdp: + fmk = Const.PT_FRAMEWORK + supported_fmk = [Const.PT_FRAMEWORK, Const.MS_FRAMEWORK] + supported_dtype_list = ["bfloat16", "float16", "float32", "float64"] + _framework = None + framework = FrameworkDescriptor() + + @classmethod + def import_framework(cls): + if cls.fmk == Const.PT_FRAMEWORK: + import torch + cls._framework = torch + elif cls.fmk == Const.MS_FRAMEWORK: + import mindspore + cls._framework = mindspore + else: + raise Exception(f"init framework adapter error, not in {cls.supported_fmk}") + + @classmethod + def set_fmk(cls, fmk=Const.PT_FRAMEWORK): + if fmk not in cls.supported_fmk: + raise Exception(f"init framework adapter error, not in {cls.supported_fmk}") + cls.fmk = fmk + cls._framework = None # 重置框架，以便下次访问时重新导入 + + @classmethod + def get_rank(cls): + if cls.fmk == Const.PT_FRAMEWORK: + return cls.framework.distributed.get_rank() + return cls.framework.communication.get_rank() + + @classmethod + def get_rank_id(cls): + if cls.is_initialized(): + return cls.get_rank() + return 0 + + @classmethod + def is_initialized(cls): + if cls.fmk == Const.PT_FRAMEWORK: + return cls.framework.distributed.is_initialized() + return cls.framework.communication.GlobalComm.INITED + + @classmethod + def is_nn_module(cls, module): + if cls.fmk == Const.PT_FRAMEWORK: + return isinstance(module, cls.framework.nn.Module) + return isinstance(module, cls.framework.nn.Cell) + + @classmethod + def is_tensor(cls, tensor): + if cls.fmk == Const.PT_FRAMEWORK: + return isinstance(tensor, cls.framework.Tensor) + return isinstance(tensor, cls.framework.Tensor) + + @classmethod + def process_tensor(cls, tensor, func): + if cls.fmk == Const.PT_FRAMEWORK: + if not tensor.is_floating_point() or tensor.dtype == cls.framework.float64: + tensor = tensor.float() + return float(func(tensor)) + return float(func(tensor).asnumpy()) + + @classmethod + def tensor_max(cls, tensor): + return cls.process_tensor(tensor, lambda x: x.max()) + + @classmethod + def tensor_min(cls, tensor): + return cls.process_tensor(tensor, lambda x: x.min()) + + @classmethod + def tensor_mean(cls, tensor): + return cls.process_tensor(tensor, lambda x: x.mean()) + + @classmethod + def tensor_norm(cls, tensor): + return cls.process_tensor(tensor, lambda x: x.norm()) + + @classmethod + def save_tensor(cls, tensor, filepath): + if cls.fmk == Const.PT_FRAMEWORK: + tensor_npy = tensor.cpu().detach().float().numpy() + else: + tensor_npy = tensor.asnumpy() + save_npy(tensor_npy, filepath) + + @classmethod + def dtype(cls, dtype_str): + if dtype_str not in cls.supported_dtype_list: + raise Exception(f"{dtype_str} is not supported by adapter, not in {cls.supported_dtype_list}") + return getattr(cls.framework, dtype_str) + + @classmethod + def named_parameters(cls, module): + if cls.fmk == Const.PT_FRAMEWORK: + if not isinstance(module, cls.framework.nn.Module): + raise Exception(f"{module} is not a torch.nn.Module") + return module.named_parameters() + if not isinstance(module, cls.framework.nn.Cell): + raise Exception(f"{module} is not a mindspore.nn.Cell") + return module.parameters_and_names() + + @classmethod + def register_forward_pre_hook(cls, module, hook, with_kwargs=False): + if cls.fmk == Const.PT_FRAMEWORK: + if not isinstance(module, cls.framework.nn.Module): + raise Exception(f"{module} is not a torch.nn.Module") + module.register_forward_pre_hook(hook, with_kwargs=with_kwargs) + else: + if not isinstance(module, cls.framework.nn.Cell): + raise Exception(f"{module} is not a mindspore.nn.Cell") + original_construct = module.construct + + @functools.wraps(original_construct) + def new_construct(*args, **kwargs): + if with_kwargs: + hook(module, args, kwargs) + else: + hook(module, args) + return original_construct(*args, **kwargs) + + module.construct = new_construct + + @classmethod + def load_checkpoint(cls, path, to_cpu=True, weights_only=True): + if cls.fmk == Const.PT_FRAMEWORK: + from msprobe.pytorch.common.utils import load_pt + return load_pt(path, to_cpu=to_cpu, weights_only=weights_only) + check_file_or_directory_path(path) + return mindspore.load_checkpoint(path) + + @classmethod + def asnumpy(cls, tensor): + if cls.fmk == Const.PT_FRAMEWORK: + return tensor.float().numpy() + return tensor.float().asnumpy() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/common/global_lock.py b/debug/accuracy_tools/msprobe/core/common/global_lock.py new file mode 100644 index 0000000000000000000000000000000000000000..2090f009ea5a78a7c5fbda61c12b6c0a842b7d25 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/common/global_lock.py @@ -0,0 +1,86 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import multiprocessing +from multiprocessing.shared_memory import SharedMemory +import random +import time +import atexit +import os + +from msprobe.core.common.log import logger + + +def is_main_process(): + return multiprocessing.current_process().name == 'MainProcess' + + +class GlobalLock: + def __init__(self): + self.name = self.get_lock_name() + try: + self._shm = SharedMemory(create=False, name=self.name) + time.sleep(random.randint(0, 500) / 10000) # 等待随机时长以避免同时获得锁 + except FileNotFoundError: + try: + self._shm = SharedMemory(create=True, name=self.name, size=1) + self._shm.buf[0] = 0 + logger.debug(f'{self.name} is created.') + except FileExistsError: + self.__init__() + + @classmethod + def get_lock_name(cls): + if is_main_process(): + return f'global_lock_{os.getpid()}' + return f'global_lock_{os.getppid()}' + + @classmethod + def is_lock_exist(cls): + try: + SharedMemory(create=False, name=cls.get_lock_name()).close() + return True + except FileNotFoundError: + return False + + def cleanup(self): + self._shm.close() + if is_main_process(): + try: + self._shm.unlink() + logger.debug(f'{self.name} is unlinked.') + except FileNotFoundError: + logger.warning(f'{self.name} has already been unlinked.') + + def acquire(self, timeout=180): + """ + acquire global lock, default timeout is 3 minutes. + + :param float timeout: timeout(seconds), default value is 180. + """ + start = time.time() + while time.time() - start < timeout: + if self._shm.buf[0] == 0: + self._shm.buf[0] = 1 + return + time.sleep(random.randint(10, 500) / 10000) # 自旋，等待1-50ms + self._shm.buf[0] = 1 + + def release(self): + self._shm.buf[0] = 0 + + +global_lock = GlobalLock() +atexit.register(global_lock.cleanup) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 963ca82545650c71f451ecc5624416a9e7b316df..d59807c77a4c6bc750146f1d7bf05eb153419f08 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -18,9 +18,8 @@ import os import re import subprocess import time -from collections import defaultdict +import inspect from datetime import datetime, timezone -from functools import wraps import numpy as np @@ -28,10 +27,15 @@ from msprobe.core.common.file_utils import (FileOpen, check_file_or_directory_pa from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.log import logger from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common.decorator import recursion_depth_decorator device = collections.namedtuple('device', ['type', 'index']) prefixes = ['api_stack', 'list', 'range', 'acl'] +file_suffix_to_file_type = { + "dump.json": Const.DUMP_JSON_FILE, + "debug.json": Const.DEBUG_JSON_FILE, +} class MsprobeBaseException(Exception): @@ -76,6 +80,7 @@ class MsprobeBaseException(Exception): NAMES_STRUCTS_MATCH_ERROR = 34 INVALID_STATE_ERROR = 35 INVALID_API_NAME_ERROR = 36 + CROSS_FRAME_ERROR = 37 def __init__(self, code, error_info: str = ""): super(MsprobeBaseException, self).__init__() @@ -192,27 +197,6 @@ def check_regex_prefix_format_valid(prefix): raise ValueError(f"prefix contains invalid characters, prefix pattern {Const.REGEX_PREFIX_PATTERN}") -def execute_command(cmd): - """ - Function Description: - run the following command - Parameter: - cmd: command - Exception Description: - when invalid command throw exception - """ - logger.info('Execute command:%s' % cmd) - process = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - while process.poll() is None: - line = process.stdout.readline() - line = line.strip() - if line: - logger.info(line) - if process.returncode != 0: - logger.error('Failed to execute command:%s' % " ".join(cmd)) - raise CompareException(CompareException.INVALID_DATA_ERROR) - - def add_time_as_suffix(name): return '{}_{}.csv'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) @@ -233,17 +217,33 @@ def format_value(value): return float('{:.12f}'.format(value)) -def md5_find(data): - for key_op in data: - for api_info in data[key_op]: - if isinstance(data[key_op][api_info], list): - for data_detail in data[key_op][api_info]: - if data_detail and 'md5' in data_detail: - return True - if isinstance(data[key_op][api_info], bool): - continue - elif data[key_op][api_info] and 'md5' in data[key_op][api_info]: +@recursion_depth_decorator('msprobe.core.common.utils.md5_find', max_depth=Const.DUMP_MAX_DEPTH) +def md5_find(data, json_type=Const.DUMP_JSON_FILE): + if json_type == Const.DUMP_JSON_FILE: + for key_op in data: + for api_info in data[key_op]: + if isinstance(data[key_op][api_info], list): + for data_detail in data[key_op][api_info]: + if data_detail and Const.MD5 in data_detail: + return True + if isinstance(data[key_op][api_info], bool): + continue + elif data[key_op][api_info] and Const.MD5 in data[key_op][api_info]: + return True + elif json_type == Const.DEBUG_JSON_FILE: + if isinstance(data, dict): + if Const.MD5 in data: return True + else: + for _, data_info in data.items(): + if md5_find(data_info, Const.DEBUG_JSON_FILE): + return True + elif isinstance(data, list): + for data_info in data: + if md5_find(data_info, Const.DEBUG_JSON_FILE): + return True + else: + return False return False @@ -281,13 +281,26 @@ def get_stack_construct_by_dump_json_path(dump_json_path): def set_dump_path(input_param): npu_path = input_param.get("npu_json_path", None) bench_path = input_param.get("bench_json_path", None) - npu_path_valid = npu_path is not None and npu_path.endswith("dump.json") - bench_path_valid = bench_path is not None and bench_path.endswith("dump.json") - if not npu_path_valid or not bench_path_valid: - logger.error(f"Please check the json path is valid. npu_path: {npu_path}, bench_path: {bench_path}") + dump_json_path_valid = npu_path is not None and npu_path.endswith("dump.json") and \ + bench_path is not None and bench_path.endswith("dump.json") + debug_json_path_valid = npu_path is not None and npu_path.endswith("debug.json") and \ + bench_path is not None and bench_path.endswith("debug.json") + if not dump_json_path_valid and not debug_json_path_valid: + logger.error(f"Please check the json path is valid and ensure that neither npu_path nor bench_path is None.") raise CompareException(CompareException.INVALID_PATH_ERROR) - input_param['npu_dump_data_dir'] = os.path.join(os.path.dirname(npu_path), Const.DUMP_TENSOR_DATA) - input_param['bench_dump_data_dir'] = os.path.join(os.path.dirname(bench_path), Const.DUMP_TENSOR_DATA) + input_param[CompareConst.NPU_DUMP_DATA_DIR] = os.path.join(os.path.dirname(npu_path), Const.DUMP_TENSOR_DATA) + input_param[CompareConst.BENCH_DUMP_DATA_DIR] = os.path.join(os.path.dirname(bench_path), Const.DUMP_TENSOR_DATA) + + +def get_file_type(file_path): + if not isinstance(file_path, str): + logger.error("get_file_type failed, check the type of file_path.") + raise CompareException(CompareException.INVALID_PATH_ERROR) + file_type = file_suffix_to_file_type.get(file_path.split(Const.SCOPE_SEPARATOR)[-1]) + if file_type is None: + logger.error("get_file_type failed, file_path is neither dump.json nor debug.json.") + raise CompareException(CompareException.INVALID_PATH_ERROR) + return file_type def get_dump_mode(input_param): @@ -295,6 +308,7 @@ def get_dump_mode(input_param): bench_path = input_param.get("bench_json_path", None) npu_json_data = load_json(npu_path) bench_json_data = load_json(bench_path) + json_type = get_file_type(file_path=npu_path) npu_task = npu_json_data.get('task', None) bench_task = bench_json_data.get('task', None) @@ -314,8 +328,8 @@ def get_dump_mode(input_param): return Const.STRUCTURE if npu_task == Const.STATISTICS: - npu_md5_compare = md5_find(npu_json_data['data']) - bench_md5_compare = md5_find(bench_json_data['data']) + npu_md5_compare = md5_find(npu_json_data['data'], json_type) + bench_md5_compare = md5_find(bench_json_data['data'], json_type) if npu_md5_compare == bench_md5_compare: return Const.MD5 if npu_md5_compare else Const.SUMMARY else: @@ -438,6 +452,28 @@ def check_init_step(step): f"{step} must be greater than or equal to 0") +def check_token_range(token_range): + if token_range is None: + return + if not isinstance(token_range, (list, tuple)): + logger.error("Token_range must be a list or tuple.") + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) + if len(token_range) != 2: + logger.error("Token_range must contains exactly 2 elements.") + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) + + start, end = token_range + if not isinstance(start, int) or not isinstance(end, int): + logger.error("Start and end in token_range must be integer.") + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) + if start > end: + logger.error("Start in token_range must less than the end.") + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) + if start < 0: + logger.error("Start in token_range must >= 0.") + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) + + def check_seed_all(seed, mode, rm_dropout): if is_int(seed): if seed < 0 or seed > Const.MAX_SEED_VALUE: @@ -481,36 +517,6 @@ def safe_get_value(container, index, container_name, key=None): raise MsprobeBaseException(MsprobeBaseException.INVALID_OBJECT_TYPE_ERROR) from e -# 记录工具函数递归的深度 -recursion_depth = defaultdict(int) - - -# 装饰一个函数，当函数递归调用超过限制时，抛出异常并打印函数信息。 -def recursion_depth_decorator(func_info, max_depth=Const.MAX_DEPTH): - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - func_id = id(func) - recursion_depth[func_id] += 1 - if recursion_depth[func_id] > max_depth: - msg = f"call {func_info} exceeds the recursion limit." - logger.error_log_with_exp( - msg, - MsprobeException( - MsprobeException.RECURSION_LIMIT_ERROR, msg - ), - ) - try: - result = func(*args, **kwargs) - finally: - recursion_depth[func_id] -= 1 - return result - - return wrapper - - return decorator - - def check_str_param(param): if not re.match(Const.REGEX_PREFIX_PATTERN, param): logger.error('The parameter {} contains special characters.'.format(param)) @@ -523,4 +529,60 @@ class DumpPathAggregation: construct_file_path = None dump_tensor_data_dir = None free_benchmark_file_path = None - debug_file_path = None \ No newline at end of file + debug_file_path = None + + +def is_save_variable_valid(variable, valid_special_types, depth=0): + if depth > Const.DUMP_MAX_DEPTH: + return False + if isinstance(variable, valid_special_types): + return True + elif isinstance(variable, (list, tuple)): + return all(is_save_variable_valid(item, valid_special_types, depth + 1) for item in variable) + elif isinstance(variable, dict): + return all(isinstance(key, str) and is_save_variable_valid(value, valid_special_types, depth + 1) + for key, value in variable.items()) + else: + return False + + +def replace_last_occurrence(text, old, new): + if text is None: + return text + index = text.rfind(old) + if index != -1: + return text[:index] + text[index:].replace(old, new, 1) + return text + + +def load_stack_json(stack_path): + stack_dict = load_json(stack_path) + if not stack_dict.get(Const.NEW_STACK_FLAG): + return stack_dict + + new_stack_dict = {} + for stack_info in stack_dict.values(): + if len(stack_info) != 2: + continue + api_list, stack_str = stack_info + for api_name in api_list: + new_stack_dict.update({api_name: stack_str}) + return new_stack_dict + + +def analyze_api_call_stack(name): + try: + api_stack = inspect.stack()[2:] + except Exception as e: + logger.warning(f"The call stack of {name} failed to retrieve, {e}.") + api_stack = None + stack_str = [] + if api_stack: + for (_, path, line, func, code, _) in api_stack: + if not code: + continue + stack_line = f"File {path}, line {str(line)}, in {func}, \n {code[0].strip()} \n" + stack_str.append(stack_line) + else: + stack_str.append(Const.WITHOUT_CALL_STACK) + return "".join(stack_str) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index b9a717c0c52f11e52ac055e3cfe6a0e77fe7e44c..836a7b89d3008c8e2fc34053eddd186e875279d6 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -111,3 +111,10 @@ class BaseConfig: f"The element '{mode}' of data_mode {self.data_mode} is not in {Const.DUMP_DATA_MODE_LIST}.", MsprobeException(MsprobeException.INVALID_PARAM_ERROR) ) + + def _check_summary_mode(self): + if self.summary_mode and self.summary_mode not in Const.SUMMARY_MODE: + logger.error_log_with_exp( + f"summary_mode is invalid, summary_mode is not in {Const.SUMMARY_MODE}.", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR) + ) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index f2aa8c479ecd0e40f3585708f82ff48a0e74832c..cabe6a7a4572a21704de5ab1dc6ec925b6e49432 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -13,111 +13,229 @@ # See the License for the specific language governing permissions and # limitations under the License. -import multiprocessing import os import re -from copy import deepcopy +from dataclasses import dataclass +from collections import defaultdict +import numpy as np import pandas as pd from tqdm import tqdm from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import load_json, remove_path +from msprobe.core.common.file_utils import load_json, remove_path, create_directory from msprobe.core.common.log import logger -from msprobe.core.common.utils import CompareException, add_time_with_xlsx, check_op_str_pattern_valid, safe_get_value -from msprobe.core.compare.check import check_dump_json_str, check_graph_mode, check_stack_json_str, \ - check_struct_match, fuzzy_check_op -from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx -from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _handle_multi_process, _save_cmp_result -from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_flag_and_msg -from msprobe.core.compare.utils import get_accuracy, get_rela_diff_summary_mode, get_un_match_accuracy, merge_tensor, \ - print_compare_ends_info, read_op, get_name_and_state, reorder_op_x_list - - -class ModeConfig: - def __init__(self, stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=None): - self.stack_mode = stack_mode - self.auto_analyze = auto_analyze - self.fuzzy_match = fuzzy_match - self.dump_mode = dump_mode +from msprobe.core.common.utils import CompareException, add_time_with_xlsx, check_op_str_pattern_valid, \ + set_dump_path, get_dump_mode, check_compare_param, check_configuration_param, load_stack_json, get_file_type +from msprobe.core.compare.check import check_dump_json_str, check_stack_json_str, cross_dtype_mapping +from msprobe.core.compare.utils import merge_tensor, print_compare_ends_info, read_op, \ + reorder_op_x_list, set_stack_json_path +from msprobe.core.compare.config import ModeConfig, MappingConfig, MappingDict +from msprobe.core.compare.multiprocessing_compute import CompareRealData +from msprobe.core.compare.highlight import HighLight + + +@dataclass +class ComparisonConfig: + dump_mode: str + stack_mode: bool + auto_analyze: bool + fuzzy_match: bool + data_mapping: dict + suffix: str + cell_mapping: dict + api_mapping: dict + layer_mapping: dict + compared_file_type: str class Comparator: - def __init__(self, mode_config: ModeConfig): - self.stack_mode = mode_config.stack_mode - self.auto_analyze = mode_config.auto_analyze - self.fuzzy_match = mode_config.fuzzy_match - self.dump_mode = mode_config.dump_mode + def __init__(self, file_reader, mode_config: ModeConfig, mapping_config: MappingConfig, is_cross_framework=False): + self.file_reader = file_reader + self.mode_config = mode_config + self.mapping_config = mapping_config + self.cross_frame = is_cross_framework + + self.mapping_dict = MappingDict(mapping_config) @staticmethod - def get_result_md5_compare(ms_op_name, bench_op_name, npu_ops_all, bench_ops_all, *args): - npu_struct = npu_ops_all.get(ms_op_name).get('struct', []) - bench_struct = bench_ops_all.get(bench_op_name).get('struct', []) + def process_output_file(output_path, suffix, compared_file_type): + file_name_prefix_mapping = { + Const.DUMP_JSON_FILE: "compare_result", + Const.DEBUG_JSON_FILE: "debug_compare_result" + } + file_name_prefix = file_name_prefix_mapping.get(compared_file_type, "compare_result") + file_name = add_time_with_xlsx(file_name_prefix + suffix) + file_path = os.path.join(os.path.realpath(output_path), file_name) + if os.path.exists(file_path): + logger.warning(f"{file_path} will be deleted.") + remove_path(file_path) + return file_path - if len(npu_struct) < 3 or len(bench_struct) < 3: - logger.error(f"The length of npu_struct and bench_struct must be >= 3, " - f"but got npu_struct={len(npu_struct)} and bench_struct={len(bench_struct)}. Please check!") - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) + def compare_core(self, input_param, output_path, **kwargs): + """ + Compares data from multiple JSON files and generates a comparison report. - result_item = [ms_op_name, bench_op_name, npu_struct[0], bench_struct[0], - npu_struct[1], bench_struct[1], npu_struct[2], bench_struct[2], - CompareConst.PASS if npu_struct[2] == bench_struct[2] else CompareConst.DIFF] + Args: + input_param (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", + "stack_path"). + output_path (str): The path where the output Excel report will be saved. + **kwargs: Additional keyword arguments including: + - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. + - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. + - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. + - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. + - dump_mode (str): ALL, SUMMARY, MD5. - if len(args) >= 2 and args[0]: - result_item.extend(args[1]) - else: - result_item.append(CompareConst.NONE) - return result_item + Returns: + """ + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - @staticmethod - def calculate_summary_data(npu_summary_data, bench_summary_data, result_item): - err_msg = "" - result_item, accuracy_check, err_msg = get_rela_diff_summary_mode(result_item, npu_summary_data, - bench_summary_data, err_msg) - result_item.append(accuracy_check) - result_item.append(err_msg) + # get kwargs or set default value + suffix = kwargs.get('suffix', '') - @staticmethod - def _generate_na_data(ops_all): - if not ops_all: - return {} - key = next(iter(ops_all)) - value = deepcopy(ops_all[key]) - for k, v in value.items(): - if isinstance(v, tuple): - value[k] = tuple(CompareConst.N_A for _ in range(len(v))) - elif isinstance(v, list): - value[k] = [CompareConst.N_A] * len(v) - else: - value[k] = CompareConst.N_A - return value + # process output file + file_path = self.process_output_file(output_path, suffix, self.mode_config.compared_file_type) - def make_result_table(self, result): - header = CompareConst.HEAD_OF_COMPARE_MODE[self.dump_mode][:] + # initialize the compare result table and compare general data(name, dtype, shape, statistics/md5, etc.) + npu_json = input_param.get("npu_json_path") + bench_json = input_param.get("bench_json_path") + stack_json = input_param.get("stack_json_path") + result_df = self.compare_statistics([npu_json, bench_json, stack_json]) + if not result_df.values.tolist(): + logger.warning("Can`t match any op.") + return - if self.stack_mode: - header.append(CompareConst.STACK) - if self.dump_mode == Const.ALL: - header.append(CompareConst.DATA_NAME) - else: - if self.dump_mode == Const.ALL: - for row in result: - del row[-2] # 输出结果不要堆栈信息时，删除中间结果result中的stack info，真实数据时为倒数第2列 - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] # 输出结果不要堆栈信息时，删除中间结果result中的stack info，非真实数据时为倒数第1列 - result_df = pd.DataFrame(result, columns=header, dtype='object') - return result_df + # compare real data + if self.mode_config.dump_mode == Const.ALL: + compare_real_data = CompareRealData(self.file_reader, self.mode_config, self.cross_frame) + result_df = compare_real_data.do_multi_process(input_param, result_df) + + # highlight suspicious API + highlight_dict = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} + highlight = HighLight(self.mode_config) + if self.mode_config.compared_file_type == Const.DUMP_JSON_FILE: + highlight.find_compare_result_error_rows(result_df, highlight_dict) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) + + # output compare analysis suggestions + if self.mode_config.auto_analyze: + advisor = Advisor(result_df, output_path, suffix) + advisor.analysis() + + print_compare_ends_info() + + def compare_statistics(self, file_list): + # load and parse json data + parse_data = ParseData(self.mode_config) + npu_df, bench_df = parse_data.parse(file_list) + + npu_df[[Const.DTYPE, Const.SHAPE]] = npu_df[[Const.DTYPE, Const.SHAPE]].astype(str) + bench_df[[Const.DTYPE, Const.SHAPE]] = bench_df[[Const.DTYPE, Const.SHAPE]].astype(str) + + # create new columns for compare op_name and shape + # process npu_df's COMPARE_KEY whether same or different framework + process_df = ProcessDf(self.mode_config, self.mapping_config, self.mapping_dict) + npu_df, bench_df = process_df.process_compare_key_and_shape(npu_df, bench_df) + + # match npu and bench, match_result contains both npu_info and bench_info + match = Match(self.mode_config, self.mapping_config, self.cross_frame) + match_result = match.match_api_infos(npu_df, bench_df) + # 筛选出npu_name存在的行并填充筛选出行中的缺失值为N/A + match_result = match_result[match_result['op_name_x'].notna()].fillna(CompareConst.N_A) + bench_columns = [i + '_y' for i in bench_df.columns] + match_result.loc[~match.gen_dtype_condition(match_result), bench_columns] = CompareConst.N_A + + # organize compare result table by renaming columns + create_table = CreateTable(self.mode_config) + result_df, header = create_table.make_result_df(match_result) + + # calculate statistics diff + calc_stats_diff = CalcStatsDiff(self.mode_config) + return calc_stats_diff.calc_accuracy(result_df, header) + + +class ParseData: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config + + def parse(self, file_list): + npu_json_path, bench_json_path, stack_json_path = file_list + npu_json_data = load_json(npu_json_path) + bench_json_data = load_json(bench_json_path) + stack_json_data = load_stack_json(stack_json_path) if self.mode_config.stack_mode else None + + # parse json data and generate df + npu_df = self.gen_data_df(npu_json_data, stack_json_data) + bench_df = self.gen_data_df(bench_json_data, stack_json_data) + + return npu_df, bench_df + + def gen_data_df(self, data_json, stack_json_data): + result = { + CompareConst.OP_NAME: [], + Const.DTYPE: [], + Const.SHAPE: [], + Const.SUMMARY: [], + Const.STACK_INFO: [] + } + if self.mode_config.dump_mode == Const.ALL: + result['data_name'] = [] + elif self.mode_config.dump_mode == Const.MD5: + result[Const.MD5] = [] + + api_nums = len(data_json['data']) + progress_bar = tqdm(total=api_nums, desc="API/Module Read Progress", unit="api/module", ncols=100) + + # 从json中循环解析API数据，遍历所有API + for data_name in data_json['data']: + check_op_str_pattern_valid(data_name) + merge_list = self.gen_merge_list(data_json, data_name, stack_json_data) + if not merge_list: + continue + + op_name_list = merge_list.get(CompareConst.OP_NAME) + summary_list = merge_list.get(Const.SUMMARY) + data_name_list = merge_list.get('data_name') + op_name_reorder, summary_reorder, data_name_reorder = reorder_op_x_list(op_name_list, + summary_list, + data_name_list) + # 遍历单个API的所有item + for index, op_name in enumerate(op_name_reorder): + result[CompareConst.OP_NAME].append(op_name) + if (CompareConst.INPUT_PATTERN in op_name) or (CompareConst.KWARGS_PATTERN in op_name): + struct = merge_list[CompareConst.INPUT_STRUCT].pop(0) + elif CompareConst.OUTPUT_PATTERN in op_name: + struct = merge_list[CompareConst.OUTPUT_STRUCT].pop(0) + elif CompareConst.PARAMS_PATTERN in op_name: + struct = merge_list[CompareConst.PARAMS_STRUCT].pop(0) + elif CompareConst.PARAMS_GRAD_PATTERN in op_name: + struct = merge_list[CompareConst.PARAMS_GRAD_STRUCT].pop(0) + else: + struct = merge_list[CompareConst.DEBUG_STRUCT].pop(0) + result[Const.DTYPE].append(struct[0]) + result[Const.SHAPE].append(struct[1]) + if self.mode_config.dump_mode == Const.MD5: + result[Const.MD5].append(struct[2]) + result[Const.SUMMARY].append(summary_reorder.pop(0)) + result[Const.STACK_INFO].append( + merge_list[Const.STACK_INFO][0] if index == 0 and self.mode_config.stack_mode else None) + if self.mode_config.dump_mode == Const.ALL: + result['data_name'].append(data_name_reorder.pop(0)) + + progress_bar.update(1) + progress_bar.close() + return pd.DataFrame(result) def gen_merge_list(self, json_data, op_name, stack_json_data): op_data = json_data['data'][op_name] - check_dump_json_str(op_data, op_name) + if self.mode_config.compared_file_type == Const.DUMP_JSON_FILE: + check_dump_json_str(op_data, op_name) op_parsed_list = read_op(op_data, op_name) - if self.stack_mode: + if self.mode_config.stack_mode: stack_info = stack_json_data.get(op_name) if stack_info is not None: check_stack_json_str(stack_info, op_name) @@ -127,387 +245,483 @@ class Comparator: 'full_info': stack_info }) - merge_list = merge_tensor(op_parsed_list, self.dump_mode) + merge_list = merge_tensor(op_parsed_list, self.mode_config.dump_mode) return merge_list - def check_op(self, npu_dict, bench_dict): - npu_op_name = npu_dict[CompareConst.OP_NAME] - bench_op_name = bench_dict[CompareConst.OP_NAME] - graph_mode = check_graph_mode(safe_get_value(npu_op_name, 0, "npu_op_name"), - safe_get_value(bench_op_name, 0, "bench_op_name")) - - frame_name = getattr(self, "frame_name") - if frame_name == "PTComparator": - from msprobe.pytorch.compare.match import graph_mapping - if graph_mode: - return graph_mapping.match(npu_op_name[0], bench_op_name[0]) - struct_match = check_struct_match(npu_dict, bench_dict) - if not self.fuzzy_match: - name_match = npu_op_name == bench_op_name - return name_match and struct_match - try: - name_match = fuzzy_check_op(npu_op_name, bench_op_name) - except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (npu_op_name, bench_op_name)) - name_match = False - return name_match and struct_match - - def match_op(self, npu_queue, bench_queue): - for b_index, b_op in enumerate(bench_queue[0: -1]): - if self.check_op(npu_queue[-1], b_op): - return len(npu_queue) - 1, b_index - if self.check_op(npu_queue[-1], bench_queue[-1]): - return len(npu_queue) - 1, len(bench_queue) - 1 - for n_index, n_op in enumerate(npu_queue[0: -1]): - if self.check_op(n_op, bench_queue[-1]): - return n_index, len(bench_queue) - 1 - return -1, -1 - def compare_process(self, file_lists): - npu_json_path, bench_json_path, stack_json_path = file_lists - npu_json_data = load_json(npu_json_path) - bench_json_data = load_json(bench_json_path) - stack_json_data = load_json(stack_json_path) if self.stack_mode else None +class ProcessDf: + def __init__(self, mode_config: ModeConfig, mapping_config: MappingConfig, mapping_dict: MappingDict): + self.mode_config = mode_config + self.mapping_config = mapping_config + self.mapping_dict = mapping_dict - if self.fuzzy_match: - logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") + @staticmethod + def get_api_name(api_list): + try: + api_name = api_list[0] + Const.SEP + api_list[1] + except IndexError as error: + logger.error('Failed to retrieve API name, please check if the dump data is reasonable') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error + return api_name + + def process_compare_key_and_shape(self, npu_df, bench_df): + npu_df = self.assign_npu_df_compare_key(npu_df, bench_df) + npu_df[CompareConst.CMP_SHAPE] = npu_df[Const.SHAPE] + bench_df[CompareConst.CMP_KEY] = bench_df[CompareConst.OP_NAME] + bench_df[CompareConst.CMP_SHAPE] = bench_df[Const.SHAPE] + return npu_df, bench_df + + def assign_npu_df_compare_key(self, npu_df, bench_df): + """ + 处理 npu_df 的 COMPARE_KEY 赋值逻辑 - npu_ops_queue = [] - bench_ops_queue = [] - result = [] + :param npu_df: DataFrame，NPU 对比数据 + :param bench_df: DataFrame，Bench 对比数据 + :return: compare_key(name)处理后的 npu_df + """ + # 处理api_mapping映射 + if self.mapping_config.api_mapping: + # 如果用户不传api_mapping.yaml，先使用内置api_mapping.yaml替换npu_op_name + npu_df[CompareConst.CMP_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_internal_api_mapping) + # 如果用户传入api_mapping.yaml，再使用传入api_mapping.yaml进一步替换npu_op_name + if isinstance(self.mapping_config.api_mapping, str): + self.modify_compare_data_with_user_mapping(npu_df, bench_df) + # 处理cell_mapping映射 + elif self.mapping_config.cell_mapping: + npu_df[CompareConst.CMP_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_cell_mapping) + # 处理data_mapping映射 + elif self.mapping_config.data_mapping: + npu_df[CompareConst.CMP_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_data_mapping) + else: + npu_df[CompareConst.CMP_KEY] = npu_df[CompareConst.OP_NAME] + return npu_df + + def process_internal_api_mapping(self, npu_op_name): + # get api name & class name from op_name + ms_api_name = self.get_api_name(npu_op_name.split(Const.SEP)) + class_name = ms_api_name.split(Const.SEP)[0] + if class_name == "Mint": + return npu_op_name.replace("Mint", "Torch") + elif class_name == "MintFunctional": + return npu_op_name.replace("MintFunctional", "Functional") + elif self.mapping_dict.ms_to_pt_mapping.get(ms_api_name): + return npu_op_name.replace(ms_api_name, self.mapping_dict.ms_to_pt_mapping.get(ms_api_name)) + else: + return npu_op_name + + def modify_compare_data_with_user_mapping(self, npu_df, bench_df): + def gen_input_compare_key(pattern, term): + is_unmatched = True + for i, prefix in enumerate(mapping_dict.get(f'ms_{term}')): + if op_name.split(pattern)[1].startswith(str(prefix)): + npu_df.loc[index, CompareConst.CMP_KEY] = ( + op_name.replace(pattern + str(prefix), + pattern + str(mapping_dict.get(f'pt_{term}')[i]))) + is_unmatched = False + return is_unmatched + + ms_api_indices_dict = self.get_api_indices_dict(npu_df) + pt_api_indices_dict = self.get_api_indices_dict(bench_df) + + for mapping_dict in self.mapping_dict.api_mapping_dict: + all_length_equal = True + for k1, k2 in CompareConst.API_MAPPING_KEYS_TO_COMPARE: + if len(mapping_dict.get(k1, [])) != len(mapping_dict.get(k2, [])): + all_length_equal = False + if not all_length_equal: + logger.warning('The user-defined mapping table is incorrect,\ + make sure that the number of parameters is equal') + continue - ops_npu_iter = iter(npu_json_data['data']) - ops_bench_iter = iter(bench_json_data['data']) - read_err_npu = True - read_err_bench = True - last_npu_ops_len = 0 - last_bench_ops_len = 0 + ms_api, pt_api = mapping_dict.get('ms_api'), mapping_dict.get('pt_api') + if ms_api not in ms_api_indices_dict or pt_api not in pt_api_indices_dict: + continue + for index in ms_api_indices_dict.get(ms_api): + op_name = npu_df.loc[index, CompareConst.OP_NAME].replace(ms_api, pt_api, 1) + if CompareConst.INPUT_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.INPUT_PATTERN, 'args') + elif CompareConst.KWARGS_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.KWARGS_PATTERN, 'args') + elif CompareConst.OUTPUT_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.OUTPUT_PATTERN, 'output') + elif CompareConst.PARAMS_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.PARAMS_PATTERN, 'parameters') + elif CompareConst.PARAMS_GRAD_PATTERN in op_name: + is_abandoned = gen_input_compare_key(CompareConst.PARAMS_GRAD_PATTERN, 'parameters_grad') + else: + logger.error(f'Excepted op_name: {op_name}') + raise CompareException(CompareException.INVALID_DATA_ERROR) + if is_abandoned: + npu_df.loc[index, CompareConst.CMP_KEY] = op_name + 'abandoned' - npu_api_nums = len(npu_json_data['data']) - progress_bar = tqdm(total=npu_api_nums, desc="API/Module Read Progress", unit="item", ncols=100) + def get_api_indices_dict(self, op_name_df): + """ + 生成多个api对应的各自的所有的input、output等的index的键值对字典 + 示例： + {'Functional.conv2d': [0, 1, 2, 3], + 'Functional.batch_norm': [4, 5, 6, 7, 8] + } + """ + api_indices_dict = defaultdict(list) + for op_index, name in enumerate(op_name_df[CompareConst.OP_NAME]): + api_name = self.get_api_name(name.split(Const.SEP)) + api_indices_dict[api_name].append(op_index) + return api_indices_dict + + def process_cell_mapping(self, npu_op_name): + if not npu_op_name: + return CompareConst.N_A + param_grad_flag = Const.PARAMS_GRAD in npu_op_name.split(Const.SEP) + if not param_grad_flag and not re.search(Const.REGEX_FORWARD_BACKWARD, npu_op_name): + return CompareConst.N_A + npu_op_name = npu_op_name.replace("Cell", "Module", 1) + if self.mapping_dict.cell_mapping_dict: + # get cell name & class name from op_name + # Cell.fc1.Dense.forward.0.input.0 + cell_name = re.split(r'\.(?:forward|backward|parameters_grad)\.', npu_op_name.split(Const.SEP, 1)[-1])[0] + if cell_name in self.mapping_dict.cell_mapping_dict: + npu_op_name = npu_op_name.replace(cell_name, self.mapping_dict.cell_mapping_dict[cell_name], 1) + return npu_op_name + + def process_data_mapping(self, npu_op_name): + return self.mapping_dict.data_mapping_dict.get(npu_op_name, npu_op_name) + + +class Match: + def __init__(self, mode_config: ModeConfig, mapping_config: MappingConfig, cross_frame): + self.mode_config = mode_config + self.mapping_config = mapping_config + self.cross_frame = cross_frame - while True: - if not read_err_npu and not read_err_bench: - break - try: - last_npu_ops_len = len(npu_ops_queue) - op_name_npu = next(ops_npu_iter) - check_op_str_pattern_valid(op_name_npu) - npu_merge_list = self.gen_merge_list(npu_json_data, op_name_npu, stack_json_data) - if npu_merge_list: - npu_ops_queue.append(npu_merge_list) - except StopIteration: - read_err_npu = False - try: - last_bench_ops_len = len(bench_ops_queue) - op_name_bench = next(ops_bench_iter) - check_op_str_pattern_valid(op_name_bench) - bench_merge_list = self.gen_merge_list(bench_json_data, op_name_bench, stack_json_data) - if bench_merge_list: - bench_ops_queue.append(bench_merge_list) - except StopIteration: - read_err_bench = False + @staticmethod + def put_unmatched_in_table(match_result, npu_op_item): + npu_columns = npu_op_item.index.tolist()[:-2] + new_columns = [name[:-1] + 'y' for name in npu_columns] + na_series = pd.Series([CompareConst.N_A] * len(new_columns), index=new_columns) + new_result_item = pd.concat([npu_op_item, na_series]).to_frame().T + new_result_item.columns = CompareConst.MATCH_RESULT_COLUMNS + match_result = pd.concat([match_result, new_result_item]) + return match_result - progress_bar.update(1) + @staticmethod + def put_matched_in_table(match_result, npu_op_item, bench_op_item): + head_len = len(CompareConst.MATCH_RESULT_COLUMNS) + new_result_item = pd.concat([npu_op_item, bench_op_item]).head(head_len).to_frame().T + new_result_item.columns = CompareConst.MATCH_RESULT_COLUMNS + match_result = pd.concat([match_result, new_result_item]) + return match_result - # merge all boolean expressions - both_empty = not npu_ops_queue and not bench_ops_queue - no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) - if both_empty or no_change: - continue + @staticmethod + def rename_api(op_name): + """ + 原api： {api_type}.{api_name}.{API调用次数}.{前向反向}.{input/output}.{参数序号} + rename后： {api_type}.{api_name}.{API调用次数}.{input/output}.{参数序号} + """ + if Const.FORWARD not in op_name and Const.BACKWARD not in op_name: + return op_name + process = Const.FORWARD if Const.FORWARD in op_name else Const.BACKWARD + name_split = op_name.split(process) + try: + torch_func_index, in_out = name_split[0], name_split[1] + except IndexError as error: + logger.error(f'{op_name} can not be split with {process}, please check!') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error + torch_func_split = torch_func_index.rsplit(Const.SEP, 2) + torch_func = str(torch_func_split[0]) + Const.SEP + process + str(in_out) + return torch_func + + def check_op_item(self, npu_op_item, bench_op_item): + name_match = self.rename_api(npu_op_item[CompareConst.CMP_KEY]) == self.rename_api( + bench_op_item[CompareConst.CMP_KEY]) + shape_match = npu_op_item[CompareConst.CMP_SHAPE] == bench_op_item[CompareConst.CMP_SHAPE] + if name_match and shape_match: + return True + else: + npu_op_name = npu_op_item[CompareConst.OP_NAME] + bench_op_name = bench_op_item[CompareConst.OP_NAME] + check_op_str_pattern_valid(npu_op_name) + check_op_str_pattern_valid(bench_op_name) + logger.warning(f"{npu_op_name} and {bench_op_name} can not fuzzy match") + return False + + def match_api_infos(self, npu_df, bench_df): + """ + 正常匹配和模糊匹配 + """ + if self.mapping_config.data_mapping: + match_result = pd.merge(npu_df, bench_df, on=[CompareConst.CMP_KEY], how='left') + + # reorder match_result by op_name of npu + op_name_order = npu_df[CompareConst.OP_NAME].tolist() + match_result[CompareConst.OP_NAME_X] = pd.Categorical(match_result[CompareConst.OP_NAME_X], + categories=op_name_order, ordered=True) + match_result = match_result.sort_values(CompareConst.OP_NAME_X).reset_index(drop=True) + match_result[CompareConst.OP_NAME_X] = match_result[CompareConst.OP_NAME_X].astype('object') + elif not self.mode_config.fuzzy_match: + match_result = pd.merge(npu_df, bench_df, on=[CompareConst.CMP_KEY, CompareConst.CMP_SHAPE], + how='outer') + else: + match_result = self.process_fuzzy_match(npu_df, bench_df) + return match_result - # APIs in NPU and Bench models unconsistent judgment + def process_fuzzy_match(self, npu_df, bench_df): + """ + 模糊匹配通过循环方式匹配api + """ + npu_ops_queue = [] + bench_ops_queue = [] + match_result = pd.DataFrame(columns=CompareConst.MATCH_RESULT_COLUMNS) + + max_len = max(len(npu_df), len(bench_df)) + min_len = min(len(npu_df), len(bench_df)) + for i in range(max_len): + if i < min_len: + npu_ops_queue.append(npu_df.iloc[i]) + bench_ops_queue.append(bench_df.iloc[i]) + else: + try: + npu_ops_queue.append(npu_df.iloc[i]) + except IndexError: + pass + try: + bench_ops_queue.append(bench_df.iloc[i]) + except IndexError: + pass + + # 如果append之后queue状态不一致，则判断结束 if bool(npu_ops_queue) ^ bool(bench_ops_queue): - logger.info("Please check whether the number and calls of APIs in NPU and Bench models are consistent.") break - n_match_point, b_match_point = self.match_op(npu_ops_queue, bench_ops_queue) + npu_match_point, bench_match_point = self.match_op(npu_ops_queue, bench_ops_queue) - # 如果没有匹配到，数据放到队列中，跳过，直到后面匹配到，把匹配之前的api放到不匹配中 - if n_match_point == -1 and b_match_point == -1: + # 如果没有匹配到，数据放到队列中，跳过。直到后面匹配到，把匹配之前的api放到不匹配中 + if npu_match_point == -1 and bench_match_point == -1: continue - n_match_data = npu_ops_queue[n_match_point] - b_match_data = bench_ops_queue[b_match_point] - un_match_data = npu_ops_queue[0: n_match_point] - for npu_data in un_match_data: - get_un_match_accuracy(result, npu_data, self.dump_mode) - get_accuracy(result, n_match_data, b_match_data, self.dump_mode) - del npu_ops_queue[0: n_match_point + 1] - del bench_ops_queue[0: b_match_point + 1] - progress_bar.close() + npu_op_item = npu_ops_queue[npu_match_point] + bench_op_item = bench_ops_queue[bench_match_point] + unmatched_data = npu_ops_queue[0: npu_match_point] + for op_item in unmatched_data: + match_result = self.put_unmatched_in_table(match_result, op_item) + match_result = self.put_matched_in_table(match_result, npu_op_item, bench_op_item) + del npu_ops_queue[0: npu_match_point + 1] + del bench_ops_queue[0: bench_match_point + 1] + if npu_ops_queue: - for npu_data in npu_ops_queue: - get_un_match_accuracy(result, npu_data, self.dump_mode) - - result_df = self.make_result_table(result) - return result_df - - def merge_data(self, json_data, stack_json_data): - ops_all = {} - for op_name in json_data.get('data', {}): - merge_list = self.gen_merge_list(json_data, op_name, stack_json_data) - if merge_list: - struct_to_index_mapping = { - CompareConst.INPUT_STRUCT: 0, - CompareConst.OUTPUT_STRUCT: 0, - CompareConst.PARAMS_STRUCT: 0, - CompareConst.PARAMS_GRAD_STRUCT: 0 - } - - op_name_list = merge_list.get(CompareConst.OP_NAME) - summary_list = merge_list.get(Const.SUMMARY) - data_name_list = merge_list.get('data_name') - op_name_reorder, summary_reorder, data_name_reorder = reorder_op_x_list(op_name_list, - summary_list, - data_name_list) - for index, op_full_name in enumerate(op_name_reorder): - data_name = data_name_reorder[index] if data_name_reorder else None - - _, state = get_name_and_state(op_full_name) - struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) - if not struct_key: - continue - ops_all[op_full_name] = { - CompareConst.STRUCT: safe_get_value(merge_list, struct_to_index_mapping.get(struct_key), - "merge_list", key=struct_key), - CompareConst.SUMMARY: safe_get_value(summary_reorder, index, "summary_reorder"), - 'data_name': data_name, - 'stack_info': merge_list.get('stack_info') - } - struct_to_index_mapping[struct_key] += 1 - return ops_all - - def get_accuracy(self, npu_ops_all, bench_ops_all): - result = [] - bench_ops_all[CompareConst.N_A] = self._generate_na_data(bench_ops_all) - for ms_op_name, bench_op_name in self.data_mapping_dict.items(): - if ms_op_name in npu_ops_all and bench_op_name in bench_ops_all: - npu_stack_info = npu_ops_all.get(ms_op_name).get("stack_info", None) - bench_stack_info = bench_ops_all.get(bench_op_name).get("stack_info", None) - has_stack = npu_stack_info and bench_stack_info - if self.dump_mode == Const.MD5: - result.append(self.get_result_md5_compare(ms_op_name, bench_op_name, npu_ops_all, - bench_ops_all, has_stack, npu_stack_info)) - continue - - npu_struct = npu_ops_all.get(ms_op_name).get('struct', []) - bench_struct = bench_ops_all.get(bench_op_name).get('struct', []) - - if len(npu_struct) < 2 or len(bench_struct) < 2: - logger.error( - f"The length of npu_struct and bench_struct must be >= 2, " - f"but got npu_struct={len(npu_struct)} and bench_struct={len(bench_struct)}. " - f"Please check!" - ) - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - - base_result_item = [ - ms_op_name, bench_op_name, - npu_struct[0], - bench_struct[0], - npu_struct[1], - bench_struct[1] - ] - - if self.dump_mode == Const.SUMMARY: - result_item = base_result_item + [" "] * 8 # 8个统计量数据情况的比对指标 - else: - result_item = base_result_item + [" "] * 6 # 6个真实数据情况的比对指标 - - npu_summary_data = npu_ops_all.get(ms_op_name).get("summary") - result_item.extend(npu_summary_data) - bench_summary_data = bench_ops_all.get(bench_op_name).get("summary") - result_item.extend(bench_summary_data) - if self.dump_mode == Const.SUMMARY: - self.calculate_summary_data(npu_summary_data, bench_summary_data, result_item) - else: - result_item.append(CompareConst.ACCURACY_CHECK_YES) - result_item.append("") - if has_stack: - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - if self.dump_mode == Const.ALL: - ms_data_name = npu_ops_all.get(ms_op_name).get("data_name", None) - pt_data_name = bench_ops_all.get(bench_op_name).get("data_name", None) - result_item.append([ms_data_name, pt_data_name]) - result.append(result_item) - elif ms_op_name not in npu_ops_all: - logger.warning(f'Can not find npu op name : `{ms_op_name}` in npu dump json file.') - elif bench_op_name not in npu_ops_all: - logger.warning(f'Can not find bench op name : `{bench_op_name}` in bench dump json file.') - return result + for op_item in npu_ops_queue: + match_result = self.put_unmatched_in_table(match_result, op_item) - def compare_process_custom(self, file_lists): - npu_json_path, bench_json_path, stack_json_path = file_lists - npu_json_data = load_json(npu_json_path) - bench_json_data = load_json(bench_json_path) - stack_json_data = load_json(stack_json_path) if self.stack_mode else None - npu_ops_all = self.merge_data(npu_json_data, stack_json_data) - bench_ops_all = self.merge_data(bench_json_data, stack_json_data) + match_result.reset_index(drop=True, inplace=True) + return match_result - result = self.get_accuracy(npu_ops_all, bench_ops_all) - result_df = self.make_result_table(result) - return result_df + def match_op(self, npu_queue, bench_queue): + for b_index, b_op in enumerate(bench_queue[0: -1]): + if self.check_op_item(npu_queue[-1], b_op): + return len(npu_queue) - 1, b_index + if self.check_op_item(npu_queue[-1], bench_queue[-1]): + return len(npu_queue) - 1, len(bench_queue) - 1 + for n_index, n_op in enumerate(npu_queue[0: -1]): + if self.check_op_item(n_op, bench_queue[-1]): + return n_index, len(bench_queue) - 1 + return -1, -1 - def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): + def gen_dtype_condition(self, match_result): """ - :param npu_op_name: excel中的NPU_Name，例如：MintFunctional.conv2d.0.forward.input.3.0 - :param bench_op_name: excel中的Bench_Name，例如：Functional.conv2d.0.forward.input.3.0 - :param op_name_mapping_dict: op_name和npy或pt文件的映射关系 - :param input_param: npu_json_path/bench_json_path/stack_json_path等参数 - :return: result_list，包含余弦相似度、最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率和错误信息 - 用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、欧式距离 - 最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率并生成错误信息 + dtype匹配条件为npu、bench的dtype一致或属于规定的映射关系 """ - error_file, relative_err, error_flag = None, None, False - - data_name_pair = op_name_mapping_dict.get(npu_op_name) - npu_data_name = data_name_pair[0] - bench_data_name = data_name_pair[1] - - if str(npu_data_name) == '-1': # 没有npu真实数据 - n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True - elif str(bench_data_name) == '-1': # 没有bench真实数据 - n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True - error_file = 'no_bench_data' - else: - npu_dir = input_param.get("npu_dump_data_dir") - bench_dir = input_param.get("bench_dump_data_dir") - try: - frame_name = getattr(self, "frame_name") - read_npy_data = getattr(self, "read_npy_data") - if frame_name == "MSComparator": - n_value = read_npy_data(npu_dir, npu_data_name) - if self.cross_frame: - b_value = read_npy_data(bench_dir, bench_data_name, load_pt_file=True) - else: - b_value = read_npy_data(bench_dir, bench_data_name) - else: - n_value = read_npy_data(npu_dir, npu_data_name) - b_value = read_npy_data(bench_dir, bench_data_name) - except IOError as error: - error_file = error.filename - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - except (FileCheckException, CompareException): - error_file = npu_data_name - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - - # 通过n_value, b_value同时得到错误标志和错误信息 - n_value, b_value, error_flag, err_msg = get_error_flag_and_msg(n_value, b_value, - error_flag=error_flag, error_file=error_file) - - result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg) - - if self.fuzzy_match and npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A: - err_msg += " Fuzzy matching data, the comparison accuracy may be affected." - result_list.append(err_msg) - return result_list + # 如果使用了data_mapping，不校验dtype，返回全True的DataFrame + if self.mapping_config.data_mapping: + return pd.Series(True, index=match_result.index) + + npu_dtype = match_result['dtype_x'] + bench_dtype = match_result['dtype_y'] + npu_dtype = self.process_cross_frame_dtype(npu_dtype) + bench_dtype = self.process_cross_frame_dtype(bench_dtype) + + equal_condition = npu_dtype == bench_dtype + match_condition = ( + (npu_dtype.isin(CompareConst.DTYPE_MATCH_GROUPS[0]) & bench_dtype.isin( + CompareConst.DTYPE_MATCH_GROUPS[0])) | + (npu_dtype.isin(CompareConst.DTYPE_MATCH_GROUPS[1]) & bench_dtype.isin( + CompareConst.DTYPE_MATCH_GROUPS[1])) + ) + return equal_condition | match_condition - def compare_core(self, input_param, output_path, **kwargs): - """ - Compares data from multiple JSON files and generates a comparison report. + def process_cross_frame_dtype(self, dtype): + if self.cross_frame: + dtype = dtype.map(cross_dtype_mapping).fillna(dtype) + return dtype - Args: - input_param (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", - "stack_path"). - output_path (str): The path where the output Excel report will be saved. - **kwargs: Additional keyword arguments including: - - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. - - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. - - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. - - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. - - dump_mode (str): ALL, SUMMARY, MD5. - Returns: - """ - # get kwargs or set default value - suffix = kwargs.get('suffix', '') +class CreateTable: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config - logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - file_name = add_time_with_xlsx("compare_result" + suffix) - file_path = os.path.join(os.path.realpath(output_path), file_name) - remove_path(file_path) - highlight_dict = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} + @staticmethod + def process_data_name(result): + result['data_name_x'] = result.apply(lambda row: [row['data_name_x'], row['data_name_y']], axis=1) + return result - npu_json = input_param.get("npu_json_path") - bench_json = input_param.get("bench_json_path") - stack_json = input_param.get("stack_json_path") - if self.data_mapping: - result_df = self.compare_process_custom([npu_json, bench_json, stack_json]) - else: - result_df = self.compare_process([npu_json, bench_json, stack_json]) + @staticmethod + def set_summary(summary): + if summary == CompareConst.N_A: + return [CompareConst.N_A] * 4 # 4为统计值个数 + summary_list = [] + for i in summary: + if str(i).lower() == 'nan': + summary_list.append(CompareConst.NAN) + else: + summary_list.append(i) + return summary_list - if not result_df.values.tolist(): - logger.warning("Can`t match any op.") - return + def make_result_df(self, result): + # get header + header = CompareConst.HEAD_OF_COMPARE_MODE[self.mode_config.dump_mode][:] + if self.mode_config.stack_mode: + header.append(CompareConst.STACK) + if self.mode_config.dump_mode == Const.ALL: + header.append(CompareConst.DATA_NAME) + result = self.process_data_name(result) + + # rename match_result columns + result.rename(columns={'op_name_x': CompareConst.NPU_NAME, + 'op_name_y': CompareConst.BENCH_NAME, + 'dtype_x': CompareConst.NPU_DTYPE, + 'dtype_y': CompareConst.BENCH_DTYPE, + 'shape_x': CompareConst.NPU_SHAPE, + 'shape_y': CompareConst.BENCH_SHAPE, + 'md5_x': CompareConst.NPU_MD5, + 'md5_y': CompareConst.BENCH_MD5, + 'data_name_x': CompareConst.DATA_NAME, + 'stack_info_x': CompareConst.STACK}, inplace=True) + + # process summary data + npu_summary = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM] + bench_summary = [CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN, + CompareConst.BENCH_NORM] + result[npu_summary] = result['summary_x'].apply(self.set_summary).tolist() + result[bench_summary] = result['summary_y'].apply(self.set_summary).tolist() + + result_df = pd.DataFrame(columns=header) + for h in header: + if h in result.columns: + result_df[h] = result[h] + return result_df, header + + +class CalcStatsDiff: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config - if self.dump_mode == Const.ALL: - result_df = self.do_multi_process(input_param, result_df) + @staticmethod + def type_check(val): + """ + 检查是否为数值或字符串形式的nan, 如果是返回True + """ + check_series = pd.Series(False, index=val.index) + val_str = val.astype(str) + check_series[pd.to_numeric(val_str, errors='coerce').notna() | val_str.str.lower().eq('nan')] = True + return check_series - find_compare_result_error_rows(result_df, highlight_dict, self.dump_mode) - highlight_rows_xlsx(result_df, highlight_dict, file_path) + @staticmethod + def get_number(val): + return pd.to_numeric(val.astype(str), errors='coerce') + + def calc_summary_diff(self, result_df, cond_no_bench, stats_index: str): + npu_val = result_df['NPU ' + stats_index] + bench_val = result_df['Bench ' + stats_index] + diff_name = stats_index.capitalize() + ' diff' + rel_err_name = ('norm' if stats_index == 'l2norm' else stats_index).capitalize() + 'RelativeErr' + + # npu、bench中统计量均为数字或nan + cond_num_nan = self.type_check(npu_val) & self.type_check(bench_val) + + # 如果统计量不是数字或nan，就赋值统计量差异为N/A + result_df.loc[~cond_num_nan, [diff_name, rel_err_name]] = CompareConst.N_A + cond_valid_stat = ~cond_no_bench & cond_num_nan # 有效统计条件：bench_name不是N/A，并且NPU和bench的统计量都是数字或nan + result_df.loc[cond_valid_stat, diff_name] = self.get_number(npu_val) - self.get_number(bench_val) + + cond_diff_nan = result_df[diff_name].isna() # 统计量差异是nan + cond_nan_diff = cond_valid_stat & cond_diff_nan + result_df.loc[cond_nan_diff, [diff_name, rel_err_name]] = CompareConst.NAN + + cond_not_nan_diff = cond_valid_stat & ~cond_diff_nan + condition_pt_zero = bench_val == 0 + result_df.loc[cond_not_nan_diff & condition_pt_zero, rel_err_name] = CompareConst.N_A + + # 相对误差转成百分比字符串 + cond_ref_err = cond_not_nan_diff & ~condition_pt_zero + result_df.loc[cond_ref_err, rel_err_name] = ( + result_df.loc[cond_ref_err, diff_name] / bench_val[cond_ref_err] * 100) + result_df.loc[cond_ref_err, rel_err_name] = (result_df.loc[cond_ref_err, rel_err_name].abs().astype(str) + '%') + + magnitude = self.get_number(result_df[diff_name]).abs() / (pd.Series( + np.maximum(self.get_number(npu_val), self.get_number(bench_val))).abs() + CompareConst.EPSILON) + return magnitude > CompareConst.MAGNITUDE + + def calc_accuracy(self, result_df, header): + # bench name N/A represents no bench data, err_msg adds "No bench data matched." + condition_no_bench = result_df[CompareConst.BENCH_NAME] == CompareConst.N_A + result_df[condition_no_bench] = result_df[condition_no_bench].fillna(CompareConst.N_A) + result_df.loc[condition_no_bench, CompareConst.ERROR_MESSAGE] = CompareConst.NO_BENCH + + if self.mode_config.dump_mode == Const.MD5: + condition_md5_equal = result_df[CompareConst.NPU_MD5] == result_df[CompareConst.BENCH_MD5] + result_df.loc[condition_md5_equal, CompareConst.RESULT] = CompareConst.PASS + result_df.loc[~condition_md5_equal & ~condition_no_bench, CompareConst.RESULT] = CompareConst.DIFF + elif self.mode_config.dump_mode == Const.SUMMARY: + warning_list = [ + self.calc_summary_diff(result_df, condition_no_bench, stats_index) + for stats_index in ['max', 'min', 'mean', 'l2norm'] + ] + warning_flag = pd.DataFrame(warning_list).any() + result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = '' + result_df.loc[warning_flag, CompareConst.RESULT] = CompareConst.WARNING + result_df.loc[warning_flag, CompareConst.ERROR_MESSAGE] = 'Need double check api accuracy.' + else: + fill_cols = [CompareConst.COSINE, CompareConst.EUC_DIST, + CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR, + CompareConst.ONE_THOUSANDTH_ERR_RATIO, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO, + CompareConst.ERROR_MESSAGE] + result_df.loc[~condition_no_bench, fill_cols] = '' + result_df.loc[~condition_no_bench, CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES + + return result_df[header] + + +def setup_comparison(input_param, output_path, **kwargs) -> ComparisonConfig: + """公共的前置处理逻辑，返回封装后的 ComparisonConfig 对象""" + try: + config = ComparisonConfig( + dump_mode='', + stack_mode=False, + auto_analyze=kwargs.get('auto_analyze', True), + fuzzy_match=kwargs.get('fuzzy_match', False), + data_mapping=kwargs.get('data_mapping', {}), + suffix=kwargs.get('suffix', ''), + cell_mapping=kwargs.get('cell_mapping', {}), + api_mapping=kwargs.get('api_mapping', {}), + layer_mapping=kwargs.get('layer_mapping', {}), + compared_file_type='', + ) - if self.auto_analyze: - advisor = Advisor(result_df, output_path, suffix) - advisor.analysis() + set_dump_path(input_param) + config.dump_mode = get_dump_mode(input_param) + config.compared_file_type = get_file_type(input_param.get("npu_json_path", None)) - print_compare_ends_info() + # set stack_mode and set "stack_json_path" in input_param + if 'stack_json_path' in input_param: + config.stack_mode = kwargs.get('stack_mode', False) + else: + config.stack_mode = set_stack_json_path(input_param) - def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): - cos_result = [] - euc_dist_result = [] - max_err_result = [] - max_relative_err_result = [] - one_thousand_err_ratio_result = [] - five_thousand_err_ratio_result = [] - err_mess = [] - - is_print_compare_log = input_param.get("is_print_compare_log") - - for i in range(len(result_df)): - npu_op_name = result_df.iloc[i, 0] - bench_op_name = result_df.iloc[i, 1] - if is_print_compare_log: - logger.info("start compare: {}".format(npu_op_name)) - - cos_sim, euc_dist, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg \ - = self.compare_by_op(npu_op_name, bench_op_name, dump_path_dict, input_param) - - if is_print_compare_log: - logger.info( - "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, \ - one_thousand_err_ratio {}, " - "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, - err_msg, one_thousand_err_ratio, five_thousand_err_ratio)) - cos_result.append(cos_sim) - euc_dist_result.append(euc_dist) - max_err_result.append(max_abs_err) - max_relative_err_result.append(max_relative_err) - one_thousand_err_ratio_result.append(one_thousand_err_ratio) - five_thousand_err_ratio_result.append(five_thousand_err_ratio) - err_mess.append(err_msg) - - cr = ComparisonResult( - cos_result=cos_result, - euc_dist_result=euc_dist_result, - max_err_result=max_err_result, - max_relative_err_result=max_relative_err_result, - one_thousand_err_ratio_result=one_thousand_err_ratio_result, - five_thousand_err_ratio_result=five_thousand_err_ratio_result, - err_msgs=err_mess - ) + check_configuration_param(config.stack_mode, config.auto_analyze, config.fuzzy_match, + input_param.get('is_print_compare_log', True)) + create_directory(output_path) + check_compare_param(input_param, output_path, config.dump_mode, config.stack_mode) - return _save_cmp_result(idx, cr, result_df, lock) + return config - def do_multi_process(self, input_param, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_param, result_df, - multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except (CompareException, FileCheckException) as error: + logger.error('Compare failed. Please check the arguments and do it again!') + raise CompareException(error.code) from error diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index 9429d7ffa1a3c1feffb0bc68f5cde777e5f8d460..a88ddb8f5e088a9f72ef2d2b721b03dbc539c385 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -14,113 +14,46 @@ # limitations under the License. from msprobe.core.common.log import logger -from msprobe.core.compare.utils import rename_api from msprobe.core.common.utils import check_op_str_pattern_valid, CompareException -from msprobe.core.common.const import CompareConst, Const - -dtype_mapping = { - "Int8": "torch.int8", - "UInt8": "torch.uint8", - "Int16": "torch.int16", - "UInt16": "torch.uint16", - "Int32": "torch.int32", - "UInt32": "torch.uint32", - "Int64": "torch.int64", - "UInt64": "torch.uint64", - "Float16": "torch.float16", - "Float32": "torch.float32", - "Float64": "torch.float64", - "Bool": "torch.bool", - "BFloat16": "torch.bfloat16", - "Complex64": "torch.complex64", - "Complex128": "torch.complex128" +from msprobe.core.common.const import Const + +cross_dtype_mapping = { + "Int8": "int", + "torch.int8": "int", + "UInt8": "int", + "torch.uint8": "int", + "Int16": "int", + "torch.int16": "int", + "UInt16": "int", + "torch.uint16": "int", + "Int32": "int", + "torch.int32": "int", + "UInt32": "int", + "torch.uint32": "int", + "Int64": "int", + "torch.int64": "int", + "UInt64": "int", + "torch.uint64": "int", + + "Float16": "float", + "torch.float16": "float", + "Float32": "float", + "torch.float32": "float", + "Float64": "float", + "torch.float64": "float", + "BFloat16": "float", + "torch.bfloat16": "float", + + "Bool": "bool", + "torch.bool": "bool", + + "Complex64": "complex", + "torch.complex64": "complex", + "Complex128": "complex", + "torch.complex128": "complex", } -def compare_op_dict_struct(npu_dict, bench_dict): - return all(npu_dict.get(key) == bench_dict.get(key) for key in CompareConst.STRUCT_COMPARE_KEY) - - -def check_struct_match(npu_dict, bench_dict): - is_match = compare_op_dict_struct(npu_dict, bench_dict) - if not is_match: - struct_match_list = [] - try: - for i, key in enumerate(CompareConst.STRUCT_COMPARE_KEY): - # 首先额外检查input_struct是否空，input_struct不可能为空 - if i == 0 and (not npu_dict.get(key, []) or not bench_dict.get(key, [])): - return False - struct_match_list.append(check_type_shape_match(npu_dict.get(key, []), bench_dict.get(key, []))) - except CompareException as error: - err_msg = f'index out of bounds error occurs in npu or bench api, please check!\n' \ - f'npu_dict: {npu_dict}' \ - f'bench_dict: {bench_dict}' - logger.error(err_msg) - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error - is_match = all(struct_match_list) - return is_match - - -def check_type_shape_match(npu_struct, bench_struct): - """ - further check dtypes with a dtype mapping list when dtypes are not entirely consistent. - """ - if len(npu_struct) != len(bench_struct): - return False - if not npu_struct and not bench_struct: - return True - - struct_match = False - for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct): - try: - npu_type = npu_type_shape[0] - npu_shape = npu_type_shape[1] - bench_type = bench_type_shape[0] - bench_shape = bench_type_shape[1] - except IndexError as error: - logger.error(f'length of npu_type_shape: {npu_type_shape} and bench_type_shape: {bench_type_shape} ' - f'should both be 2, please check!') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error - shape_match = npu_shape == bench_shape - type_match = ((npu_type == bench_type) or - any(npu_type in group and bench_type in group for group in CompareConst.DTYPE_MATCH_GROUPS)) - struct_match = shape_match and type_match - if not struct_match: - return False - return struct_match - - -def check_graph_mode(a_op_name, b_op_name): - if Const.ATEN in a_op_name and Const.ATEN not in b_op_name: - return True - if Const.ATEN not in a_op_name and Const.ATEN in b_op_name: - return True - return False - - -def fuzzy_check_op(npu_name_list, bench_name_list): - # 先检查api里的item长度是否相等，如果不是parameters_grad, 必然有input或者output，长度不可能为0 - # 如果是parameters_grad, "parameters_grad"字段的字典不会是空字典，因此len>=1 - if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list): - return False - is_match = True - for npu_name, bench_name in zip(npu_name_list, bench_name_list): - is_match = fuzzy_check_name(npu_name, bench_name) - if not is_match: - break - return is_match - - -def fuzzy_check_name(npu_name, bench_name): - if Const.FORWARD in npu_name and Const.FORWARD in bench_name: - is_match = rename_api(npu_name, Const.FORWARD) == rename_api(bench_name, Const.FORWARD) - elif Const.BACKWARD in npu_name and Const.BACKWARD in bench_name: - is_match = rename_api(npu_name, Const.BACKWARD) == rename_api(bench_name, Const.BACKWARD) - else: - is_match = npu_name == bench_name - return is_match - - def check_dump_json_str(op_data, op_name): input_list = op_data.get(Const.INPUT_ARGS, None) if op_data.get(Const.INPUT_ARGS, None) else op_data.get( Const.INPUT, None) diff --git a/debug/accuracy_tools/msprobe/core/compare/config.py b/debug/accuracy_tools/msprobe/core/compare/config.py new file mode 100644 index 0000000000000000000000000000000000000000..448139b8b3cf545cac53a573594f7b105ddb0c41 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/compare/config.py @@ -0,0 +1,72 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.file_utils import load_yaml + + +class ModeConfig: + def __init__(self, stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.SUMMARY, + compared_file_type=Const.DUMP_JSON_FILE): + self.stack_mode = stack_mode + self.auto_analyze = auto_analyze + self.fuzzy_match = fuzzy_match + self.dump_mode = dump_mode + self.compared_file_type = compared_file_type + + +class MappingConfig: + def __init__(self, cell_mapping=None, api_mapping=None, data_mapping=None): + self.cell_mapping = cell_mapping + self.api_mapping = api_mapping + self.data_mapping = data_mapping + + +class MappingDict: + def __init__(self, mapping_config: MappingConfig): + self.cell_mapping_dict = self.load_mapping_file(mapping_config.cell_mapping) + self.api_mapping_dict = self.load_mapping_file(mapping_config.api_mapping) + if mapping_config.api_mapping is not None: + self.ms_to_pt_mapping = self.load_internal_api() + self.data_mapping_dict = self.init_data_mapping(mapping_config.data_mapping) + + @staticmethod + def load_internal_api(): + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.abspath(os.path.join(cur_path, CompareConst.INTERNAL_API_MAPPING_FILE)) + return load_yaml(yaml_path) + + @staticmethod + def load_mapping_file(mapping_file): + if isinstance(mapping_file, str): + mapping_dict = load_yaml(mapping_file) + else: + mapping_dict = {} + return mapping_dict + + def init_data_mapping(self, data_mapping): + """ + 初始化data_mapping_dict + """ + if isinstance(data_mapping, str) or data_mapping is None: + data_mapping_dict = self.load_mapping_file(data_mapping) + elif isinstance(data_mapping, dict): + data_mapping_dict = data_mapping + else: + raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " + f"{type(data_mapping)}") + return data_mapping_dict \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 1983313249f34680a8f25c3a2466d8871fe0a693..71959d77d1ad3f3e293b103c6844d9641c9e51be 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -30,12 +30,7 @@ from msprobe.core.common.file_utils import save_workbook from msprobe.core.common.log import logger from msprobe.core.common.utils import get_header_index, safe_get_value from msprobe.core.compare.utils import table_value_is_valid, get_name_and_state, CompareException - - -class HighlightCheck(abc.ABC): - @abc.abstractmethod - def apply(self, info, color_columns, dump_mode): - raise NotImplementedError +from msprobe.core.compare.config import ModeConfig def add_highlight_row_info(color_list, num, highlight_err_msg): @@ -46,6 +41,12 @@ def add_highlight_row_info(color_list, num, highlight_err_msg): color_list.append((num, [highlight_err_msg])) +class HighlightCheck(abc.ABC): + @abc.abstractmethod + def apply(self, info, color_columns, dump_mode): + raise NotImplementedError + + class CheckOrderMagnitude(HighlightCheck): """检查Max diff的数量级差异""" @@ -75,12 +76,12 @@ class CheckOneThousandErrorRatio(HighlightCheck): if (api_in[one_thousand_index] > CompareConst.ONE_THOUSAND_ERROR_IN_RED and api_out[one_thousand_index] < CompareConst.ONE_THOUSAND_ERROR_OUT_RED): add_highlight_row_info(color_columns.red, num, - "The input/parameters's one thousandth err ratio exceeds 0.9, " + "The input/parameter's one thousandth err ratio exceeds 0.9, " "while the output's is below 0.6") elif api_in[one_thousand_index] - api_out[one_thousand_index] > CompareConst.ONE_THOUSAND_ERROR_DIFF_YELLOW: add_highlight_row_info(color_columns.yellow, num, "The output's one thousandth err ratio decreases by more than 0.1 " - "compared to the input/parameters's") + "compared to the input/parameter's") class CheckCosineSimilarity(HighlightCheck): @@ -94,7 +95,7 @@ class CheckCosineSimilarity(HighlightCheck): if api_in[cosine_index] - api_out[cosine_index] > CompareConst.COSINE_DIFF_YELLOW: add_highlight_row_info(color_columns.yellow, num, "The output's cosine decreases by more than 0.1 " - "compared to the input/parameters's") + "compared to the input/parameter's") class CheckMaxRelativeDiff(HighlightCheck): @@ -117,7 +118,7 @@ class CheckMaxRelativeDiff(HighlightCheck): input_max_relative_diff < CompareConst.MAX_RELATIVE_IN_YELLOW): add_highlight_row_info(color_columns.yellow, num, "The output's maximum relative error exceeds 0.1, " - "while the input/parameters's is below 0.01") + "while the input/parameter's is below 0.01") class CheckOverflow(HighlightCheck): @@ -159,73 +160,6 @@ class HighlightRules: } -def check_indices_numeric(api_items, indices: list): - """检查指定索引处的值是否都为数字类型（int 或 float）""" - return all(isinstance(api_items[i], (float, int)) for i in indices) - - -def apply_comparison_rules(api_info, dump_mode, color_columns): - """output与input/params的比较""" - if dump_mode == Const.SUMMARY: - for rule in HighlightRules.summary_compare_rules.values(): - rule.apply(api_info, color_columns, dump_mode) - else: - for rule in HighlightRules.compare_rules.values(): - rule.apply(api_info, color_columns, dump_mode) - - -def find_error_rows(result, api_batch, highlight_dict, dump_mode): - """找到单个API中需要高亮的行""" - if dump_mode == Const.MD5: - return - npu_max_index = get_header_index(CompareConst.NPU_MAX, dump_mode) - bench_max_index = get_header_index(CompareConst.BENCH_MAX, dump_mode) - max_diff_index = get_header_index(CompareConst.MAX_DIFF if dump_mode == Const.SUMMARY - else CompareConst.MAX_ABS_ERR, dump_mode) - - red_lines, yellow_lines = [], [] - LineInfo = namedtuple('LineInfo', ['line_data', 'num_pointer']) - ApiInfo = namedtuple('ApiInfo', ['api_input', 'api_output', 'num_pointer']) - ColorColumns = namedtuple('ColorColumns', ['red', 'yellow']) - color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) - - api_batch_start = api_batch.start # result_df的input起始全局索引 - api_batch_params_end_index = api_batch.params_end_index # result_df的params结束全局索引 + 1 - api_batch_output_end_index = api_batch.output_end_index # result_df的output结束全局索引 + 1 - api_batch_params_slice_index_local = api_batch_params_end_index - api_batch_start # result的params结束局部切片索引 - api_batch_output_slice_index_local = api_batch_output_end_index - api_batch_start # result的output结束局部切片索引 - - # 对单行API的输入或输出进行误差判断 - for i, line in enumerate(result): - index = api_batch_start + i - line_info = LineInfo(line_data=line, num_pointer=index) - for rule in HighlightRules.basic_rules.values(): - rule.apply(line_info, color_columns, dump_mode) - - # 对API的输出与输入比较，进行误差判断 - for n, api_out in enumerate(result[api_batch_params_slice_index_local: api_batch_output_slice_index_local]): - index = api_batch_start + api_batch_params_slice_index_local + n - # 单行检查只有溢出检查（红色），如果已经溢出，不进一步检查 - if index in red_lines: - continue - if not check_indices_numeric(api_out, [npu_max_index, bench_max_index, max_diff_index]): - continue - - # input/parameters的比较检查, 这里api_in包括input、parameters - for _, api_in in enumerate(result[0: api_batch_params_slice_index_local]): - if not check_indices_numeric(api_in, [npu_max_index, bench_max_index, max_diff_index]): - continue - api_info = ApiInfo(api_input=api_in, api_output=api_out, num_pointer=index) - apply_comparison_rules(api_info, dump_mode, color_columns) - - red_lines_num_set = {x[0] for x in red_lines} - yellow_lines_num_set = {x[0] for x in yellow_lines} - highlight_dict.get('red_rows', set()).update(red_lines_num_set) - highlight_dict.get('yellow_rows', set()).update(yellow_lines_num_set - red_lines_num_set) - highlight_dict.get('red_lines', []).extend(red_lines) - highlight_dict.get('yellow_lines', []).extend(yellow_lines) - - class ApiBatch: def __init__(self, api_name: str, start: int): self.api_name = api_name @@ -259,159 +193,225 @@ class ApiBatch: self.params_grad_end_index += 1 -def api_batches_update(api_batches, api_name, state, index): - """ - 当一个api的所有item更新完后，input, output的索引范围： - input: [start: start+input_len] - output: [start+input_len: output_end_index] - params: [output_end_index: params_end_index] - """ - if not api_batches: - api_batches.append(ApiBatch(api_name, index)) - else: - api_batch = api_batches[-1] - if api_batch.api_name == api_name or ( - not re.search(Const.REGEX_FORWARD_BACKWARD, api_name) and api_name in api_batch.api_name): - try: - api_batch.increment(state) - except ValueError as e: - logger.error(f"api_batch: {api_batch} with invalid state, please check! {e}") - raise CompareException(CompareException.INVALID_STATE_ERROR) from e - else: - api_batches.append(ApiBatch(api_name, index)) +class HighLight: + def __init__(self, mode_config: ModeConfig): + self.mode_config = mode_config - -def find_compare_result_error_rows(result_df, highlight_dict, dump_mode): - """将dataframe根据API分组，并找到有误差的算子用于高亮""" - result = result_df.values - api_batches = [] - for i, res_i in enumerate(result): - api_full_name = safe_get_value(res_i, 0, "res_i") - api_name, state = get_name_and_state(api_full_name) - api_batches_update(api_batches, api_name, state, i) - with tqdm(total=len(api_batches), desc="API/Module Analyse Progress", unit="item", ncols=100) as progress_bar: - for api_batch in api_batches: - find_error_rows(result[api_batch.start: api_batch.params_grad_end_index], api_batch, highlight_dict, - dump_mode) - progress_bar.update(1) - - -def value_check(value, api_name=None, i=None, result_df_columns=None): - if not table_value_is_valid(value): - if result_df_columns: - logger.error(f"Malicious value [{value}] at api_name [{api_name}], column [{result_df_columns[i]}], " - f"is not allowed to be written into the compare result xlsx.") + @staticmethod + def api_batches_update(api_batches, api_name, state, index): + """ + 当一个api的所有item更新完后，input, output的索引范围： + input: [start: start+input_len] + output: [start+input_len: output_end_index] + params: [output_end_index: params_end_index] + """ + if not api_batches: + api_batches.append(ApiBatch(api_name, index)) else: - logger.error(f"Malicious value [{value}] is not allowed to be written into the compare result xlsx.") - - -def df_malicious_value_check(df_chunk, result_df_columns): - for row in df_chunk.itertuples(index=False): - api_name = row[0] - for i, value in enumerate(row): - value_check(value, api_name, i, result_df_columns) - - -def handle_multi_process_malicious_value_check(func, result_df): - result_total_nums = len(result_df) - process_num = int((multiprocessing.cpu_count() + 1) / 2) - - if result_total_nums <= process_num: - process_num = 1 - chunks = [result_df] - else: - chunk_size = result_total_nums // process_num - chunks = [result_df.iloc[i: i + chunk_size] for i in range(0, result_total_nums, chunk_size)] - - pool = multiprocessing.Pool(process_num) - - def err_call(args): - logger.error("Multiprocessing malicious value check failed! Reason: {}".format(args)) - try: - pool.terminate() - except OSError: - logger.error("Pool terminate failed") - - result_df_columns = result_df.columns.tolist() - for column in result_df_columns: - value_check(column) - for df_chunk in chunks: - pool.apply_async(func, args=(df_chunk, result_df_columns,), error_callback=err_call) - - pool.close() - pool.join() - - -def compare_result_df_convert(value): - if not isinstance(value, (float, int)) or isinstance(value, bool): # bool类型或者非数字类型转str - value = f"{str(value)}\t" if str(value) in ("inf", "-inf", "nan") else str(value) - if isinstance(value, float): - value = f"{str(value)}\t" if str(value) in ("inf", "-inf", "nan") else value - return value - - -def highlight_rows_xlsx(result_df, highlight_dict, file_path): - """Write and highlight results in Excel""" + api_batch = api_batches[-1] + if api_batch.api_name == api_name or ( + not re.search(Const.REGEX_FORWARD_BACKWARD, api_name) and api_name in api_batch.api_name): + try: + api_batch.increment(state) + except ValueError as e: + logger.error(f"api_batch: {api_batch} with invalid state, please check! {e}") + raise CompareException(CompareException.INVALID_STATE_ERROR) from e + else: + api_batches.append(ApiBatch(api_name, index)) + + @staticmethod + def check_indices_numeric(api_items, indices: list): + """检查指定索引处的值是否都为数字类型（int 或 float）""" + return all(isinstance(api_items[i], (float, int)) for i in indices) + + @staticmethod + def update_highlight_err_msg(result_df, highlight_dict): + if result_df.shape[1] <= 1: + return - update_highlight_err_msg(result_df, highlight_dict) # add highlight err_msg + if CompareConst.NPU_MD5 in result_df.columns: + return - wb = openpyxl.Workbook() - ws = wb.active + err_msg = result_df.get(CompareConst.ERROR_MESSAGE) + red_lines_num_set = highlight_dict.get('red_rows') + + for color in ['red', 'yellow']: + line_key = f'{color}_lines' + lines = highlight_dict.get(line_key, []) + for line_index, messages in lines: + if color == 'yellow' and line_index in red_lines_num_set: + continue # 如果是 yellow 行，且已被 red 行覆盖，跳过 + + for msg in messages: + if err_msg[line_index] == '': + err_msg[line_index] = msg + else: + err_msg[line_index] += '\n' + msg + + if color == 'red': + red_lines_num_set.add(line_index) + + result_df[CompareConst.ERROR_MESSAGE] = err_msg + + @staticmethod + def compare_result_df_convert(value): + if not isinstance(value, (float, int)) or isinstance(value, bool): # bool类型或者非数字类型转str + value = f"{str(value)}\t" if str(value) in ("inf", "-inf", "nan") else str(value) + if isinstance(value, float): + value = f"{str(value)}\t" if str(value) in ("inf", "-inf", "nan") else value + return value + + @staticmethod + def value_check(value, api_name=None, i=None, result_df_columns=None): + if not table_value_is_valid(value): + if result_df_columns: + logger.error(f"Malicious value [{value}] at api_name [{api_name}], column [{result_df_columns[i]}], " + f"is not allowed to be written into the compare result xlsx.") + else: + logger.error(f"Malicious value [{value}] is not allowed to be written into the compare result xlsx.") + + def find_compare_result_error_rows(self, result_df, highlight_dict): + """将dataframe根据API分组，并找到有误差的算子用于高亮""" + result = result_df.values + api_batches = [] + for i, res_i in enumerate(result): + api_full_name = safe_get_value(res_i, 0, "res_i") + api_name, state = get_name_and_state(api_full_name) + self.api_batches_update(api_batches, api_name, state, i) + with tqdm(total=len(api_batches), desc="API/Module Analyse Progress", unit="item", ncols=100) as progress_bar: + for api_batch in api_batches: + self.find_error_rows(result[api_batch.start: api_batch.params_grad_end_index], api_batch, + highlight_dict) + progress_bar.update(1) + + def find_error_rows(self, result, api_batch, highlight_dict): + """找到单个API中需要高亮的行""" + if self.mode_config.dump_mode == Const.MD5: + return + npu_max_index = get_header_index(CompareConst.NPU_MAX, self.mode_config.dump_mode) + bench_max_index = get_header_index(CompareConst.BENCH_MAX, self.mode_config.dump_mode) + max_diff_index = get_header_index(CompareConst.MAX_DIFF if self.mode_config.dump_mode == Const.SUMMARY + else CompareConst.MAX_ABS_ERR, self.mode_config.dump_mode) + + red_lines, yellow_lines = [], [] + LineInfo = namedtuple('LineInfo', ['line_data', 'num_pointer']) + ApiInfo = namedtuple('ApiInfo', ['api_input', 'api_output', 'num_pointer']) + ColorColumns = namedtuple('ColorColumns', ['red', 'yellow']) + color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) + + api_batch_start = api_batch.start # result_df的input起始全局索引 + api_batch_params_end_index = api_batch.params_end_index # result_df的params结束全局索引 + 1 + api_batch_output_end_index = api_batch.output_end_index # result_df的output结束全局索引 + 1 + api_batch_params_slice_index_local = api_batch_params_end_index - api_batch_start # result的params结束局部切片索引 + api_batch_output_slice_index_local = api_batch_output_end_index - api_batch_start # result的output结束局部切片索引 + + # 对单行API的输入或输出进行误差判断 + for i, line in enumerate(result): + index = api_batch_start + i + line_info = LineInfo(line_data=line, num_pointer=index) + for rule in HighlightRules.basic_rules.values(): + rule.apply(line_info, color_columns, self.mode_config.dump_mode) + + # 对API的输出与输入比较，进行误差判断 + for n, api_out in enumerate(result[api_batch_params_slice_index_local: api_batch_output_slice_index_local]): + index = api_batch_start + api_batch_params_slice_index_local + n + # 单行检查只有溢出检查（红色），如果已经溢出，不进一步检查 + if index in red_lines: + continue + if not self.check_indices_numeric(api_out, [npu_max_index, bench_max_index, max_diff_index]): + continue - # write header - logger.info('Initializing Excel file.') + # input/parameters的比较检查, 这里api_in包括input、parameters + for api_in in result[0: api_batch_params_slice_index_local]: + if not self.check_indices_numeric(api_in, [npu_max_index, bench_max_index, max_diff_index]): + continue + api_info = ApiInfo(api_input=api_in, api_output=api_out, num_pointer=index) + self.apply_comparison_rules(api_info, color_columns) + + red_lines_num_set = {x[0] for x in red_lines} + yellow_lines_num_set = {x[0] for x in yellow_lines} + highlight_dict.get('red_rows', set()).update(red_lines_num_set) + highlight_dict.get('yellow_rows', set()).update(yellow_lines_num_set - red_lines_num_set) + highlight_dict.get('red_lines', []).extend(red_lines) + highlight_dict.get('yellow_lines', []).extend(yellow_lines) + + def apply_comparison_rules(self, api_info, color_columns): + """output与input/params的比较""" + if self.mode_config.dump_mode == Const.SUMMARY: + for rule in HighlightRules.summary_compare_rules.values(): + rule.apply(api_info, color_columns, self.mode_config.dump_mode) + else: + for rule in HighlightRules.compare_rules.values(): + rule.apply(api_info, color_columns, self.mode_config.dump_mode) - handle_multi_process_malicious_value_check(df_malicious_value_check, result_df) + def highlight_rows_xlsx(self, result_df, highlight_dict, file_path): + """Write and highlight results in Excel""" - result_df_convert = result_df.applymap(compare_result_df_convert) + self.update_highlight_err_msg(result_df, highlight_dict) # add highlight err_msg - for row in dataframe_to_rows(result_df_convert, index=False, header=True): - ws.append(row) + wb = openpyxl.Workbook() + ws = wb.active - # 对可疑数据标色 - logger.info('Coloring Excel in progress.') - col_len = len(result_df.columns) - red_fill = PatternFill( - start_color=CompareConst.RED, end_color=CompareConst.RED, fill_type="solid" - ) - yellow_fill = PatternFill( - start_color=CompareConst.YELLOW, end_color=CompareConst.YELLOW, fill_type="solid", - ) - for i in highlight_dict.get("red_rows", []): - for j in range(1, col_len + 1): - ws.cell(row=i + 2, column=j).fill = red_fill # 2因为ws.cell中的row或column需要>=1,数据从第2行开始 - for i in highlight_dict.get("yellow_rows", []): - for j in range(1, col_len + 1): - ws.cell(row=i + 2, column=j).fill = yellow_fill + # write header + logger.info('Initializing Excel file.') - logger.info('Saving Excel file to disk: %s' % file_path) - save_workbook(wb, file_path) + self.handle_multi_process_malicious_value_check(self.df_malicious_value_check, result_df) + result_df_convert = result_df.applymap(self.compare_result_df_convert) -def update_highlight_err_msg(result_df, highlight_dict): - if result_df.shape[1] <= 1: - return + for row in dataframe_to_rows(result_df_convert, index=False, header=True): + ws.append(row) - if CompareConst.NPU_MD5 in result_df.columns: - return + # 对可疑数据标色 + logger.info('Coloring Excel in progress.') + col_len = len(result_df.columns) + red_fill = PatternFill( + start_color=CompareConst.RED, end_color=CompareConst.RED, fill_type="solid" + ) + yellow_fill = PatternFill( + start_color=CompareConst.YELLOW, end_color=CompareConst.YELLOW, fill_type="solid", + ) + for i in highlight_dict.get("red_rows", []): + for j in range(1, col_len + 1): + ws.cell(row=i + 2, column=j).fill = red_fill # 2因为ws.cell中的row或column需要>=1,数据从第2行开始 + for i in highlight_dict.get("yellow_rows", []): + for j in range(1, col_len + 1): + ws.cell(row=i + 2, column=j).fill = yellow_fill - err_msg = result_df.get(CompareConst.ERROR_MESSAGE) - red_lines_num_set = highlight_dict.get('red_rows') + logger.info('Saving Excel file to disk: %s' % file_path) + save_workbook(wb, file_path) - for color in ['red', 'yellow']: - line_key = f'{color}_lines' - lines = highlight_dict.get(line_key, []) - for line_index, messages in lines: - if color == 'yellow' and line_index in red_lines_num_set: - continue # 如果是 yellow 行，且已被 red 行覆盖，跳过 + def handle_multi_process_malicious_value_check(self, func, result_df): + result_total_nums = len(result_df) + process_num = int((multiprocessing.cpu_count() + 1) / 2) - for msg in messages: - if err_msg[line_index] == '': - err_msg[line_index] = msg - else: - err_msg[line_index] += '\n' + msg + if result_total_nums <= process_num: + process_num = 1 + chunks = [result_df] + else: + chunk_size = result_total_nums // process_num + chunks = [result_df.iloc[i: i + chunk_size] for i in range(0, result_total_nums, chunk_size)] - if color == 'red': - red_lines_num_set.add(line_index) + pool = multiprocessing.Pool(process_num) - result_df[CompareConst.ERROR_MESSAGE] = err_msg + def err_call(args): + logger.error("Multiprocessing malicious value check failed! Reason: {}".format(args)) + try: + pool.close() + except OSError: + logger.error("Pool terminate failed") + + result_df_columns = result_df.columns.tolist() + for column in result_df_columns: + self.value_check(column) + for df_chunk in chunks: + pool.apply_async(func, args=(df_chunk, result_df_columns,), error_callback=err_call) + + pool.close() + pool.join() + + def df_malicious_value_check(self, df_chunk, result_df_columns): + for row in df_chunk.itertuples(index=False): + api_name = row[0] + for i, value in enumerate(row): + self.value_check(value, api_name, i, result_df_columns) diff --git a/debug/accuracy_tools/msprobe/core/compare/layer_mapping/layer_mapping.py b/debug/accuracy_tools/msprobe/core/compare/layer_mapping/layer_mapping.py index d0f19462ee1ccf4d72c69885c18174cec32df056..59cd8db909cb8205280db0b702013ef75796ba92 100644 --- a/debug/accuracy_tools/msprobe/core/compare/layer_mapping/layer_mapping.py +++ b/debug/accuracy_tools/msprobe/core/compare/layer_mapping/layer_mapping.py @@ -23,7 +23,7 @@ from msprobe.core.common.utils import (add_time_with_yaml, get_stack_construct_by_dump_json_path) from msprobe.core.compare.layer_mapping.data_scope_parser import get_dump_data_items from msprobe.core.compare.utils import read_op, reorder_op_name_list - +from msprobe.core.common.decorator import recursion_depth_decorator class LayerTrie: @@ -71,6 +71,7 @@ class LayerTrie: file_path = os.path.join(os.path.realpath(output_path), file_name) save_yaml(file_path, result) + @recursion_depth_decorator("LayerMapping: LayerTrie.convert_to_dict", max_depth=100) def convert_to_dict(self, node): result = {} result["data_item"] = {st: [dt.data_name for dt in dts] for st, dts in node.data_items.items()} diff --git a/debug/accuracy_tools/msprobe/core/compare/merge_result/merge_result.py b/debug/accuracy_tools/msprobe/core/compare/merge_result/merge_result.py index b605bd59fca0b2b3a510a7a686caa94383488bd2..9edc6d9a9dc36d05325c5af98f18a296f3627e2f 100644 --- a/debug/accuracy_tools/msprobe/core/compare/merge_result/merge_result.py +++ b/debug/accuracy_tools/msprobe/core/compare/merge_result/merge_result.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,8 @@ from functools import partial import pandas as pd from tqdm import tqdm -from msprobe.core.common.file_utils import load_yaml, logger, FileChecker, save_excel, read_xlsx, create_directory +from msprobe.core.common.file_utils import load_yaml, logger, FileChecker, save_excel, read_xlsx, create_directory, \ + remove_path from msprobe.core.common.const import FileCheckConst, Const, CompareConst from msprobe.core.common.utils import CompareException, add_time_with_xlsx from msprobe.core.compare.utils import table_value_is_valid @@ -32,8 +33,8 @@ def check_compare_result_name(file_name): """ check whether the compare result name is as expected """ - single_rank_pattern = r"^compare_result_rank-rank_\d{14}.xlsx$" - multi_ranks_pattern = r"^compare_result_rank(\d+)-rank\1_\d{14}.xlsx$" + single_rank_pattern = r"^compare_result_(rank|rank-rank)_\d{14}\.xlsx$" + multi_ranks_pattern = r"^compare_result_rank(\d+)(?:-rank\1)?_\d{14}\.xlsx$" if re.match(multi_ranks_pattern, file_name): return True if re.match(single_rank_pattern, file_name): @@ -47,7 +48,7 @@ def reorder_path(compare_result_path_list): """ reorder compare results by rank num """ - rank_pattern = r"compare_result_rank(\d+)-rank" + rank_pattern = r"compare_result_rank(\d+)" reorder_path_list = sorted( compare_result_path_list, key=lambda path: int(re.search(rank_pattern, os.path.basename(path)).group(1)) @@ -63,6 +64,7 @@ def get_result_path(input_dir): for f in os.listdir(input_dir) if f.endswith(FileCheckConst.XLSX_SUFFIX)] filt_compare_result_path_list = [] for file_path in compare_result_path_list: + FileChecker(file_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check() file_name = os.path.basename(file_path) if check_compare_result_name(file_name): compare_result_path_checker = FileChecker(file_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE) @@ -236,7 +238,7 @@ def handle_multi_process(func, func_args, lock): def err_call(args): logger.error('Multiprocess merge result failed! Reason: {}'.format(args)) try: - pool.terminate() + pool.close() except OSError: logger.error("Pool terminate failed") @@ -329,6 +331,10 @@ def generate_merge_result(all_compare_index_dict_list, all_rank_num_list, all_co for i, df in enumerate(merge_df_list): # merge_df_list中df与compare_index_list中compare_index一一对应 final_result_df_list.append((df, compare_index_list[i])) + + if os.path.exists(output_path): + logger.warning(f"{output_path} will be deleted.") + remove_path(output_path) save_excel(output_path, final_result_df_list) logger.info(f"The compare results of the multi-ranks are merged and saved in: {output_path}.") diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_to_pt_api.yaml b/debug/accuracy_tools/msprobe/core/compare/ms_to_pt_api.yaml similarity index 100% rename from debug/accuracy_tools/msprobe/mindspore/compare/ms_to_pt_api.yaml rename to debug/accuracy_tools/msprobe/core/compare/ms_to_pt_api.yaml diff --git a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py index 71b0f29d64f717adc87b74cf48e891652e9e753f..510e9fd01be89c6f9c64657c7c45774f010226e2 100644 --- a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py @@ -23,48 +23,20 @@ from tqdm import tqdm from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException from msprobe.core.common.const import CompareConst +from msprobe.core.common.exceptions import FileCheckException +from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_flag_and_msg +from msprobe.core.compare.config import ModeConfig -def _handle_multi_process(func, input_param, result_df, lock): - process_num = max(int((multiprocessing.cpu_count() + 1) // 4), 1) - op_name_mapping_dict = read_dump_data(result_df) - - df_chunk_size = len(result_df) // process_num - if df_chunk_size > 0: - df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] - else: - df_chunks = [result_df] - - results = [] - pool = multiprocessing.Pool(process_num) - - def err_call(args): - logger.error('multiprocess compare failed! Reason: {}'.format(args)) - try: - pool.terminate() - except OSError as e: - logger.error("pool terminate failed") - - progress_bar = tqdm(total=len(result_df), desc="API/Module Item Compare Process", unit="row", ncols=100) - - def update_progress(size, progress_lock, extra_param=None): - with progress_lock: - progress_bar.update(size) - - for process_idx, df_chunk in enumerate(df_chunks): - idx = df_chunk_size * process_idx - chunk_size = len(df_chunk) - result = pool.apply_async(func, - args=(idx, op_name_mapping_dict, df_chunk, lock, input_param), - error_callback=err_call, - callback=partial(update_progress, chunk_size, lock) - ) - results.append(result) - - final_results = [r.get() for r in results] - pool.close() - pool.join() - return pd.concat(final_results, ignore_index=True) +@dataclass +class ComparisonResult: + cos_result: list + euc_dist_result: list + max_err_result: list + max_relative_err_result: list + one_thousand_err_ratio_result: list + five_thousand_err_ratio_result: list + err_msgs: list def _ms_graph_handle_multi_process(func, result_df, mode): @@ -81,9 +53,9 @@ def _ms_graph_handle_multi_process(func, result_df, mode): def err_call(args): logger.error('multiprocess compare failed! Reason: {}'.format(args)) try: - pool.terminate() + pool.close() except OSError as e: - logger.error("pool terminate failed") + logger.error(f'pool terminate failed: {str(e)}') for df_chunk in df_chunks: result = pool.apply_async(func, args=(df_chunk, mode), error_callback=err_call) @@ -94,74 +66,6 @@ def _ms_graph_handle_multi_process(func, result_df, mode): return pd.concat(final_results, ignore_index=True) -def read_dump_data(result_df): - try: - npu_dump_name_list = result_df.iloc[0:, 0].tolist() - dump_tensor_pair_list = result_df.iloc[0:, -1].tolist() - op_name_mapping_dict = {} - for index, _ in enumerate(npu_dump_name_list): - npu_dump_name = npu_dump_name_list[index] - dump_tensor_pair = dump_tensor_pair_list[index] - op_name_mapping_dict[npu_dump_name] = dump_tensor_pair - return op_name_mapping_dict - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - - -@dataclass -class ComparisonResult: - cos_result: list - euc_dist_result: list - max_err_result: list - max_relative_err_result: list - one_thousand_err_ratio_result: list - five_thousand_err_ratio_result: list - err_msgs: list - - -def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): - """ - Save comparison results into the result DataFrame with thread safety. - Args: - offset: offset for index - result: data struct of ComparisonResult - result_df: result of DataFrame - lock: thread lock - - Returns: - comparison results in DataFrame - """ - - lock.acquire() - try: - for i, _ in enumerate(result.cos_result): - process_index = i + offset - result_df.loc[process_index, CompareConst.COSINE] = result.cos_result[i] - result_df.loc[process_index, CompareConst.EUC_DIST] = result.euc_dist_result[i] - result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] - result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] - result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = ( - result.one_thousand_err_ratio_result)[i] - result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = ( - result.five_thousand_err_ratio_result)[i] - result_df.loc[process_index, CompareConst.ACCURACY] = ( - check_accuracy(result.cos_result[i], result.max_err_result[i])) - result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - finally: - lock.release() - - def check_accuracy(cos, max_abs_err): if cos == CompareConst.SHAPE_UNMATCH: return CompareConst.ACCURACY_CHECK_UNMATCH @@ -179,3 +83,212 @@ def check_accuracy(cos, max_abs_err): if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: return CompareConst.ACCURACY_CHECK_NO return CompareConst.ACCURACY_CHECK_YES + + +class CompareRealData: + def __init__(self, file_reader, mode_config: ModeConfig, cross_frame): + self.file_reader = file_reader + self.mode_config = mode_config + self.cross_frame = cross_frame + + @staticmethod + def read_dump_data(result_df): + try: + npu_dump_name_list = result_df.iloc[0:, 0].tolist() + dump_tensor_pair_list = result_df.iloc[0:, -1].tolist() + op_name_mapping_dict = {} + for index, npu_dump_name in enumerate(npu_dump_name_list): + dump_tensor_pair = dump_tensor_pair_list[index] + op_name_mapping_dict[npu_dump_name] = dump_tensor_pair + return op_name_mapping_dict + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + + @staticmethod + def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): + """ + Save comparison results into the result DataFrame with thread safety. + Args: + offset: offset for index + result: data struct of ComparisonResult + result_df: result of DataFrame + lock: thread lock + + Returns: + comparison results in DataFrame + """ + + lock.acquire() + try: + for i, cos_item in enumerate(result.cos_result): + process_index = i + offset + result_df.loc[process_index, CompareConst.COSINE] = cos_item + result_df.loc[process_index, CompareConst.EUC_DIST] = result.euc_dist_result[i] + result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] + result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] + result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = ( + result.one_thousand_err_ratio_result)[i] + result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = ( + result.five_thousand_err_ratio_result)[i] + result_df.loc[process_index, CompareConst.ACCURACY] = ( + check_accuracy(result.cos_result[i], result.max_err_result[i])) + result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + finally: + lock.release() + + def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): + """ + :param npu_op_name: excel中的NPU_Name，例如：MintFunctional.conv2d.0.forward.input.3.0 + :param bench_op_name: excel中的Bench_Name，例如：Functional.conv2d.0.forward.input.3.0 + :param op_name_mapping_dict: op_name和npy或pt文件的映射关系 + :param input_param: npu_json_path/bench_json_path/stack_json_path等参数 + :return: result_list，包含余弦相似度、最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率和错误信息 + 用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、欧式距离 + 最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率并生成错误信息 + """ + error_file, relative_err, error_flag = None, None, False + + data_name_pair = op_name_mapping_dict.get(npu_op_name) + npu_data_name = data_name_pair[0] + bench_data_name = data_name_pair[1] + + if str(npu_data_name) == CompareConst.NO_REAL_DATA_FLAG: # 没有npu真实数据 + n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True + elif str(bench_data_name) == CompareConst.NO_REAL_DATA_FLAG: # 没有bench真实数据 + n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True + error_file = 'no_bench_data' + elif str(bench_data_name) == CompareConst.N_A: # bench没匹配 + n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True + error_file = None + else: + npu_dir = input_param.get(CompareConst.NPU_DUMP_DATA_DIR) + bench_dir = input_param.get(CompareConst.BENCH_DUMP_DATA_DIR) + try: + n_value, b_value = self.file_reader(npu_dir, npu_data_name, bench_dir, bench_data_name, + self.cross_frame) + except IOError as error: + error_file = error.filename + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + except (FileCheckException, CompareException): + error_file = npu_data_name + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + + # 通过n_value, b_value同时得到错误标志和错误信息 + n_value, b_value, error_flag, err_msg = get_error_flag_and_msg(n_value, b_value, + error_flag=error_flag, error_file=error_file) + + result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg) + + if self.mode_config.fuzzy_match and npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A: + err_msg += " Fuzzy matching data, the comparison accuracy may be affected." + result_list.append(err_msg) + return result_list + + def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): + cos_result = [] + euc_dist_result = [] + max_err_result = [] + max_relative_err_result = [] + one_thousand_err_ratio_result = [] + five_thousand_err_ratio_result = [] + err_mess = [] + + is_print_compare_log = input_param.get("is_print_compare_log") + + for i in range(len(result_df)): + npu_op_name = result_df.iloc[i, 0] + bench_op_name = result_df.iloc[i, 1] + if is_print_compare_log: + logger.info("start compare: {}".format(npu_op_name)) + + cos_sim, euc_dist, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg \ + = self.compare_by_op(npu_op_name, bench_op_name, dump_path_dict, input_param) + + if is_print_compare_log: + logger.info( + "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, \ + one_thousand_err_ratio {}, " + "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, + err_msg, one_thousand_err_ratio, five_thousand_err_ratio)) + cos_result.append(cos_sim) + euc_dist_result.append(euc_dist) + max_err_result.append(max_abs_err) + max_relative_err_result.append(max_relative_err) + one_thousand_err_ratio_result.append(one_thousand_err_ratio) + five_thousand_err_ratio_result.append(five_thousand_err_ratio) + err_mess.append(err_msg) + + cr = ComparisonResult( + cos_result=cos_result, + euc_dist_result=euc_dist_result, + max_err_result=max_err_result, + max_relative_err_result=max_relative_err_result, + one_thousand_err_ratio_result=one_thousand_err_ratio_result, + five_thousand_err_ratio_result=five_thousand_err_ratio_result, + err_msgs=err_mess + ) + + return self._save_cmp_result(idx, cr, result_df, lock) + + def do_multi_process(self, input_param, result_df): + try: + result_df = self._handle_multi_process(self.compare_ops, input_param, result_df, + multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + def _handle_multi_process(self, func, input_param, result_df, lock): + process_num = max(int((multiprocessing.cpu_count() + 1) // 4), 1) + op_name_mapping_dict = self.read_dump_data(result_df) + + df_chunk_size = len(result_df) // process_num + if df_chunk_size > 0: + df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] + else: + df_chunks = [result_df] + + results = [] + pool = multiprocessing.Pool(process_num) + + def err_call(args): + logger.error('multiprocess compare failed! Reason: {}'.format(args)) + try: + pool.close() + except OSError: + logger.error("pool terminate failed") + + progress_bar = tqdm(total=len(result_df), desc="API/Module Item Compare Process", unit="row", ncols=100) + + def update_progress(size, progress_lock, extra_param=None): + with progress_lock: + progress_bar.update(size) + + for process_idx, df_chunk in enumerate(df_chunks): + idx = df_chunk_size * process_idx + chunk_size = len(df_chunk) + result = pool.apply_async(func, + args=(idx, op_name_mapping_dict, df_chunk, lock, input_param), + error_callback=err_call, + callback=partial(update_progress, chunk_size, lock) + ) + results.append(result) + + final_results = [r.get() for r in results] + pool.close() + pool.join() + return pd.concat(final_results, ignore_index=True) diff --git a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py index 4103d361fec14284fc38f97e1418e5405e939cd9..b58d2854ef1c3ffcb62f144a1c3101f38efbd55b 100644 --- a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py @@ -290,10 +290,8 @@ class CompareOps: def error_value_process(n_value): - if n_value == CompareConst.READ_NONE or n_value == CompareConst.UNREADABLE: + if n_value in [CompareConst.READ_NONE, CompareConst.UNREADABLE, CompareConst.NONE]: return CompareConst.UNSUPPORTED, "" - if n_value == CompareConst.NONE: - return 0, "" if n_value == CompareConst.SHAPE_UNMATCH: return CompareConst.SHAPE_UNMATCH, "" if n_value == CompareConst.NAN: diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 66dc9ba94ee168f2ea7dbba15f4c76d5e6ef6f13..229a85aa8d603d14341ce3fcbd12a599bd9b01c1 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -20,33 +20,45 @@ import zlib from dataclasses import dataclass import numpy as np +import pandas as pd from msprobe.core.common.const import Const, CompareConst, FileCheckConst from msprobe.core.common.utils import CompareException, check_regex_prefix_format_valid, logger, safe_get_value from msprobe.core.common.file_utils import check_file_or_directory_path +json_file_mapping = { + Const.DUMP_JSON_FILE: "dump.json", + Const.DEBUG_JSON_FILE: "debug.json", + Const.STACK_JSON_FILE: "stack.json" +} -def extract_json(dirname, stack_json=False): + +def extract_json(dirname, json_file_type): json_path = '' for filename in os.listdir(dirname): - target_file_name = 'stack.json' if stack_json else 'dump.json' + target_file_name = json_file_mapping.get(json_file_type) + if target_file_name is None: + logger.error(f'extract_json failed, invalid json_file_type: {json_file_type}.') + raise CompareException(CompareException.INVALID_KEY_ERROR) if filename == target_file_name: json_path = os.path.join(dirname, filename) break # Provide robustness on invalid directory inputs if not json_path: - if stack_json: + if json_file_type == Const.STACK_JSON_FILE: logger.warning(f'stack.json is not found in dump dir {dirname}.') - else: + elif json_file_type == Const.DUMP_JSON_FILE: logger.error(f'dump.json is not found in dump dir {dirname}.') - raise CompareException(CompareException.NO_DUMP_FILE_ERROR) + elif json_file_type == Const.DEBUG_JSON_FILE: + logger.warning(f'debug.json is not found in dump dir {dirname}.') + return json_path def set_stack_json_path(input_param): npu_data_dir = os.path.dirname(input_param.get("npu_json_path")) - stack_path = extract_json(npu_data_dir, stack_json=True) + stack_path = extract_json(npu_data_dir, json_file_type=Const.STACK_JSON_FILE) input_param["stack_json_path"] = stack_path if stack_path else None return bool(stack_path) @@ -81,24 +93,9 @@ def check_and_return_dir_contents(dump_dir, prefix): return contents -def rename_api(npu_name, process): - """ - 原api： {api_type}.{api_name}.{API调用次数}.{前向反向}.{input/output}.{参数序号} - rename后： {api_type}.{api_name}.{input/output}.{参数序号} - """ - npu_split = npu_name.split(process) - try: - torch_func_index, in_out = npu_split[0], npu_split[1] - except IndexError as error: - logger.error(f'{npu_name} can not be split with {process}, please check!') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error - torch_func_split = torch_func_index.rsplit(Const.SEP, 2) - torch_func = str(torch_func_split[0]) + str(in_out) - return torch_func - - def read_op(op_data, op_name): - if Const.PARAMS_GRAD in op_name.split(Const.SEP): + split_name = op_name.split(Const.SEP) + if Const.DEBUG in split_name or Const.PARAMS_GRAD in split_name: op_parsed_list = op_item_parse(op_data, op_name) else: op_parsed_list = [] @@ -191,35 +188,152 @@ def gen_op_item(op_data, op_name): return op_item -def resolve_api_special_parameters(data_dict, full_op_name, item_list): +@dataclass +class ApiItemInfo: + name: str + struct: tuple + stack_info: list + + +def merge_tensor(tensor_list, dump_mode): + keys = [ + CompareConst.OP_NAME, + CompareConst.INPUT_STRUCT, + CompareConst.KWARGS_STRUCT, + CompareConst.OUTPUT_STRUCT, + CompareConst.PARAMS_STRUCT, + CompareConst.PARAMS_GRAD_STRUCT, + CompareConst.DEBUG_STRUCT, + Const.SUMMARY, + Const.STACK_INFO + ] + op_dict = {key: [] for key in keys} + + if dump_mode == Const.ALL: + op_dict["data_name"] = [] + + for tensor in tensor_list: + # A dict(len=2) with 'full_op_name' and 'full_info' is added to the tensor only if self.stack_mode is True + if len(tensor) == 2: + op_dict[Const.STACK_INFO].append(tensor['full_info']) + break + + op_dict[CompareConst.OP_NAME].append(tensor['full_op_name']) + + _, state = get_name_and_state(tensor['full_op_name']) + struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) + if not struct_key: + continue + if dump_mode == Const.MD5: + op_dict.get(struct_key).append((tensor[Const.DTYPE], tensor[Const.SHAPE], tensor[Const.MD5])) + else: + op_dict.get(struct_key).append((tensor[Const.DTYPE], tensor[Const.SHAPE])) + op_dict[Const.SUMMARY].append([tensor[Const.MAX], tensor[Const.MIN], tensor[Const.MEAN], tensor[Const.NORM]]) + + if dump_mode == Const.ALL: + op_dict["data_name"].append(tensor['data_name']) + + if not op_dict[CompareConst.KWARGS_STRUCT]: + del op_dict[CompareConst.KWARGS_STRUCT] + return op_dict if op_dict[CompareConst.OP_NAME] else {} + + +def print_compare_ends_info(): + total_len = len(CompareConst.COMPARE_ENDS_SUCCESSFULLY) + Const.FILL_CHAR_NUMS + logger.info('*' * total_len) + logger.info(f"*{CompareConst.COMPARE_ENDS_SUCCESSFULLY.center(total_len - 2)}*") + logger.info('*' * total_len) + + +def table_value_is_valid(value: str) -> bool: + if not isinstance(value, str): + return True + try: + # -1.00 or +1.00 should be considered as digit numbers + float(value) + except ValueError: + # otherwise, they will be considered as formular injections + return not bool(re.compile(FileCheckConst.CSV_BLACK_LIST).search(value)) + return True + + +def get_name_and_state(name): """ - Function Description: - 解析下面格式的数据, 是api参数的一种特殊格式 - { - "last_hidden_state": { - "type": "torch.Tensor", - "dtype": "torch.bfloat16", - ... - }, - "loss": { - "type": "torch.Tensor", - "dtype": "torch.float32", - ... - } - } - Parameter: - data_dict: 字典格式的数据 - full_op_name: 参数的全名字符串 - item_list: 参数信息集合 + Get api/module name and state + example: + name = 'conv2d.forward.1.input.0' + return: ('conv2d.forward.1.', 'input') + + name = 'Functional.pad.0.backward.output.0' + return: ('Functional.pad.0.backward.', 'output') + + name = 'x_tensor.0.debug.{index}' + return: ('x_tensor.0.', 'debug') + + state type: input, output, kwargs, parameters, parameters_grad, debug """ - for key, value in data_dict.items(): - if isinstance(value, dict): - parsed_item = value - parts = full_op_name.split(Const.SEP) - parts.insert(-1, key) - full_op_name_new = ".".join(parts) - parsed_item['full_op_name'] = full_op_name_new - item_list.append(parsed_item) + if not isinstance(name, str): + logger.error(f'Invalid name: {name}, type should be string, please check.') + raise CompareException(CompareException.INVALID_API_NAME_ERROR) + + if Const.DEBUG in name.split(Const.SEP): + return name.split(Const.DEBUG)[0], Const.DEBUG + if Const.PARAMS_GRAD in name.split(Const.SEP): + return name.split(Const.PARAMS_GRAD)[0], Const.PARAMS_GRAD + + split = re.split(Const.REGEX_FORWARD_BACKWARD, name) + if len(split) < 3: + logger.error(f'Invalid name string: {name}, can not be split by forward/backward, please check.') + raise CompareException(CompareException.INVALID_API_NAME_ERROR) + api = f'{split[0]}.{split[1]}.' + state_str = split[2] + match = re.match(r'^(\d+\.)?(input|output|kwargs|parameters)\..+$', state_str) + if not match: + raise CompareException(f'Invalid name string: {name}') + if match.group(1): + api = f'{api}{match.group(1)}' + state = match.group(2) + return api, state + + +def reorder_op_name_list(op_name_list): + if not op_name_list: + return op_name_list + + parameters = [] + output = [] + parameters_grad = [] + others = [] + for x in op_name_list: + state = get_name_and_state(x)[1] + if state == Const.PARAMS: + parameters.append(x) + elif state == Const.OUTPUT: + output.append(x) + elif state == Const.PARAMS_GRAD: + parameters_grad.append(x) + else: + others.append(x) + # 合并others, parameters, 和output，确保parameters排在output前面 + op_name_reorder = others + parameters + output + parameters_grad + return op_name_reorder + + +def reorder_op_x_list(op_name_list, summary_list, data_name_list): + """对op_name, summary, data_name重新排序，把parameters放到input后output前，data_name由于统计量比对时，为None，单独处理""" + if not op_name_list or not summary_list: + return op_name_list, summary_list, data_name_list + + index_map = {name: index for index, name in enumerate(op_name_list)} + + op_name_reorder = reorder_op_name_list(op_name_list) + summary_reorder = [summary_list[index_map.get(name)] for name in op_name_reorder] + if data_name_list: + data_name_reorder = [data_name_list[index_map.get(name)] for name in op_name_reorder] + else: + data_name_reorder = data_name_list + + return op_name_reorder, summary_reorder, data_name_reorder def process_summary_data(summary_data): @@ -407,204 +521,23 @@ def get_accuracy(result, n_dict, b_dict, dump_mode): CompareConst.PARAMS_GRAD_STRUCT) -def append_stack_info(result_item, npu_stack_info, index): - """添加堆栈信息到 result_item""" - if npu_stack_info and index == 0: - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - - -def get_un_match_accuracy(result, n_dict, dump_mode): - npu_stack_info = n_dict.get("stack_info", None) - bench_name, bench_type, bench_shape = CompareConst.N_A, CompareConst.N_A, CompareConst.N_A +def make_result_table(result, dump_mode, stack_mode): + header = CompareConst.HEAD_OF_COMPARE_MODE[dump_mode][:] - struct_to_index_mapping = { - CompareConst.INPUT_STRUCT: 0, - CompareConst.OUTPUT_STRUCT: 0, - CompareConst.PARAMS_STRUCT: 0, - CompareConst.PARAMS_GRAD_STRUCT: 0 - } - - op_name_list = n_dict.get(CompareConst.OP_NAME) - summary_list = n_dict.get(Const.SUMMARY) - data_name_list = n_dict.get('data_name') - op_name_reorder, summary_reorder, _ = reorder_op_x_list(op_name_list, - summary_list, - data_name_list) - for index, n_name in enumerate(op_name_reorder): - _, state = get_name_and_state(n_name) - struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) - if not struct_key: - continue - n_struct = safe_get_value(n_dict, struct_to_index_mapping.get(struct_key), "n_dict", key=struct_key) - struct_to_index_mapping[struct_key] += 1 - - try: - result_item = [n_name, bench_name, n_struct[0], bench_type, n_struct[1], bench_shape] - except IndexError as e: - err_msg = "index out of bounds error occurs, please check!\n" \ - f"op_name of n_dict is {n_dict['op_name']}\n" \ - f"input_struct of n_dict is {n_dict[CompareConst.INPUT_STRUCT]}\n" \ - f"output_struct of n_dict is {n_dict[CompareConst.OUTPUT_STRUCT]}" - logger.error(err_msg) - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - - if dump_mode == Const.MD5: - result_item.extend([CompareConst.N_A] * 3) - append_stack_info(result_item, npu_stack_info, index) - result.append(result_item) - continue - if dump_mode == Const.SUMMARY: - result_item.extend([CompareConst.N_A] * 8) # 8个统计量数据情况的比对指标 + if stack_mode: + header.append(CompareConst.STACK) if dump_mode == Const.ALL: - result_item.extend([CompareConst.N_A] * 6) # 6个真实数据情况的比对指标 - - npu_summary_data = safe_get_value(summary_reorder, index, "summary_reorder") - bench_summary_data = [CompareConst.N_A] * 4 - result_item.extend(npu_summary_data) - result_item.extend(bench_summary_data) - err_msg = CompareConst.NO_BENCH - accuracy_check_res = CompareConst.N_A - result_item.append(accuracy_check_res) - result_item.append(err_msg) - append_stack_info(result_item, npu_stack_info, index) - if dump_mode == Const.ALL and result_item[1] == CompareConst.N_A: - result_item.extend([["-1", "-1"]]) - result.append(result_item) - - -def merge_tensor(tensor_list, dump_mode): - op_dict = {} - op_dict["op_name"] = [] - op_dict[CompareConst.INPUT_STRUCT] = [] - op_dict[CompareConst.KWARGS_STRUCT] = [] - op_dict[CompareConst.OUTPUT_STRUCT] = [] - op_dict[CompareConst.PARAMS_STRUCT] = [] - op_dict[CompareConst.PARAMS_GRAD_STRUCT] = [] - op_dict[Const.SUMMARY] = [] - op_dict["stack_info"] = [] - - if dump_mode == Const.ALL: - op_dict["data_name"] = [] - - for tensor in tensor_list: - # A dict(len=2) with 'full_op_name' and 'full_info' is added to the tensor only if self.stack_mode is True - if len(tensor) == 2: - op_dict['stack_info'].append(tensor['full_info']) - break - - op_dict["op_name"].append(tensor['full_op_name']) - - _, state = get_name_and_state(tensor['full_op_name']) - struct_key = CompareConst.STATE_TO_STRUCT_MAPPING.get(state) - if not struct_key: - continue - if dump_mode == Const.MD5: - op_dict.get(struct_key).append((tensor[Const.DTYPE], tensor[Const.SHAPE], tensor[Const.MD5])) - else: - op_dict.get(struct_key).append((tensor[Const.DTYPE], tensor[Const.SHAPE])) - op_dict[Const.SUMMARY].append([tensor[Const.MAX], tensor[Const.MIN], tensor[Const.MEAN], tensor[Const.NORM]]) - + header.append(CompareConst.DATA_NAME) + else: if dump_mode == Const.ALL: - op_dict["data_name"].append(tensor['data_name']) - - if not op_dict[CompareConst.KWARGS_STRUCT]: - del op_dict[CompareConst.KWARGS_STRUCT] - return op_dict if op_dict["op_name"] else {} - - -def print_compare_ends_info(): - total_len = len(CompareConst.COMPARE_ENDS_SUCCESSFULLY) + Const.FILL_CHAR_NUMS - logger.info('*' * total_len) - logger.info(f"*{CompareConst.COMPARE_ENDS_SUCCESSFULLY.center(total_len - 2)}*") - logger.info('*' * total_len) - - -def table_value_is_valid(value: str) -> bool: - if not isinstance(value, str): - return True - try: - # -1.00 or +1.00 should be consdiered as digit numbers - float(value) - except ValueError: - # otherwise, they will be considered as formular injections - return not bool(re.compile(FileCheckConst.CSV_BLACK_LIST).search(value)) - return True - - -def get_name_and_state(name): - """ - Get api/module name and state - example: - name = 'conv2d.forward.1.input.0' - return: ('conv2d.forward.1.', 'input') - - name = 'Functional.pad.0.backward.output.0' - return: ('Functional.pad.0.backward.', 'output') - - state type: input, output, kwargs, parameters, parameters_grad - """ - if not isinstance(name, str): - logger.error(f'Invalid name: {name}, type should be string, please check.') - raise CompareException(CompareException.INVALID_API_NAME_ERROR) - - if Const.PARAMS_GRAD in name.split(Const.SEP): - return name.split(Const.PARAMS_GRAD)[0], Const.PARAMS_GRAD - - split = re.split(Const.REGEX_FORWARD_BACKWARD, name) - if len(split) < 3: - logger.error(f'Invalid name string: {name}, can not be split by forward/backward, please check.') - raise CompareException(CompareException.INVALID_API_NAME_ERROR) - api = f'{split[0]}.{split[1]}.' - state_str = split[2] - match = re.match(r'^(\d+\.)?(input|output|kwargs|parameters)\..+$', state_str) - if not match: - raise CompareException(f'Invalid name string: {name}') - if match.group(1): - api = f'{api}{match.group(1)}' - state = match.group(2) - return api, state - - -def reorder_op_name_list(op_name_list): - if not op_name_list: - return op_name_list - - parameters = [] - output = [] - parameters_grad = [] - others = [] - for x in op_name_list: - state = get_name_and_state(x)[1] - if state == Const.PARAMS: - parameters.append(x) - elif state == Const.OUTPUT: - output.append(x) - elif state == Const.PARAMS_GRAD: - parameters_grad.append(x) + for row in result: + del row[-2] # 输出结果不要堆栈信息时，删除中间结果result中的stack info，真实数据时为倒数第2列 + header.append(CompareConst.DATA_NAME) else: - others.append(x) - # 合并others, parameters, 和output，确保parameters排在output前面 - op_name_reorder = others + parameters + output + parameters_grad - return op_name_reorder - - -def reorder_op_x_list(op_name_list, summary_list, data_name_list): - """对op_name, summary, data_name重新排序，把parameters放到input后output前，data_name由于统计量比对时，为None，单独处理""" - if not op_name_list or not summary_list: - return op_name_list, summary_list, data_name_list - - index_map = {name: index for index, name in enumerate(op_name_list)} - - op_name_reorder = reorder_op_name_list(op_name_list) - summary_reorder = [summary_list[index_map.get(name)] for name in op_name_reorder] - if data_name_list: - data_name_reorder = [data_name_list[index_map.get(name)] for name in op_name_reorder] - else: - data_name_reorder = data_name_list - - return op_name_reorder, summary_reorder, data_name_reorder + for row in result: + del row[-1] # 输出结果不要堆栈信息时，删除中间结果result中的stack info，非真实数据时为倒数第1列 + result_df = pd.DataFrame(result, columns=header, dtype='object') + return result_df def _compare_parser(parser): @@ -627,3 +560,34 @@ def _compare_parser(parser): help=" The data mapping file path.", required=False) parser.add_argument("-lm", "--layer_mapping", dest="layer_mapping", type=str, nargs='?', const=True, help=" The layer mapping file path.", required=False) + + +def compare_distributed_inner(npu_dump_dir, bench_dump_dir, output_path, compare_func, **kwargs): + if kwargs.get('suffix'): + logger.error("Argument 'suffix' is not supported for compare_distributed.") + raise CompareException(CompareException.INVALID_PARAM_ERROR) + is_print_compare_log = kwargs.get('is_print_compare_log', True) + # get the ranks and match by order + npu_ranks = sorted(check_and_return_dir_contents(npu_dump_dir, 'rank')) + bench_ranks = sorted(check_and_return_dir_contents(bench_dump_dir, 'rank')) + if len(npu_ranks) != len(bench_ranks): + logger.error('The number of ranks in the two runs are different. ' + 'Unable to match the ranks. Please use another folder to compare ' + 'or use compare() api and manually match the ranks.') + raise CompareException(CompareException.INVALID_PATH_ERROR) + for nr, br in zip(npu_ranks, bench_ranks): + npu_data_dir = os.path.join(npu_dump_dir, nr) + bench_data_dir = os.path.join(bench_dump_dir, br) + for file_type in [Const.DUMP_JSON_FILE, Const.DEBUG_JSON_FILE]: + npu_path = extract_json(npu_data_dir, file_type) + bench_path = extract_json(bench_data_dir, file_type) + if npu_path == "" or bench_path == "": + logger.debug(f'Did not find paired {file_type} in {npu_data_dir} and {bench_data_dir},' + ' skip comparing.') + continue + dump_result_param = { + 'npu_json_path': npu_path, + 'bench_json_path': bench_path, + 'is_print_compare_log': is_print_compare_log + } + compare_func(input_param=dump_result_param, output_path=output_path, suffix=f'_{nr}', **kwargs) diff --git a/debug/accuracy_tools/msprobe/core/config_check/__init__.py b/debug/accuracy_tools/msprobe/core/config_check/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..621122ffa00ba40a868853ccb46ff582c3e5fdda --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import msprobe.core.config_check.checkers +from msprobe.core.config_check.config_checker import ConfigChecker diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/__init__.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9b9024b862f1f60655d2f71a47ab401546a86076 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__all__ = ['BaseChecker', 'apply_patches'] + +import msprobe.core.config_check.checkers.env_args_checker +import msprobe.core.config_check.checkers.pip_checker +import msprobe.core.config_check.checkers.dataset_checker +import msprobe.core.config_check.checkers.weights_checker +import msprobe.core.config_check.checkers.hyperparameter_checker +import msprobe.core.config_check.checkers.random_checker + +from msprobe.core.config_check.checkers.base_checker import BaseChecker diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/base_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/base_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..7f17e7c14c63eb767ae5819098499b2a2ee202c5 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/base_checker.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from msprobe.core.common.framework_adapter import FmkAdp +from msprobe.core.common.const import FileCheckConst + + +class PackInput: + + def __init__(self, output_zip_path, model, shell_path): + self.output_zip_path = output_zip_path + self.shell_path = shell_path + self.model = model[0] if isinstance(model, list) else model + self.check_input_params() + + def check_input_params(self): + if self.model and not FmkAdp.is_nn_module(self.model): + raise Exception(f"model is not torch.nn.Module/mindspore.nn.Cell or module list.") + if not isinstance(self.output_zip_path, str) or not self.output_zip_path.endswith(FileCheckConst.ZIP_SUFFIX): + raise Exception(f"output zip path must be a string and ends with '.zip'") + + +class BaseChecker: + input_needed = None + target_name_in_zip = None + multi_rank = False + + @staticmethod + def pack(pack_input): + pass + + @staticmethod + def compare(bench_dir, cmp_dir, output_path, fmk): + pass + + @staticmethod + def apply_patches(fmk): + pass + + @classmethod + def compare_ex(cls, bench_dir, cmp_dir, output_path, fmk): + bench_filepath = os.path.join(bench_dir, cls.target_name_in_zip) + cmp_filepath = os.path.join(cmp_dir, cls.target_name_in_zip) + if not os.path.exists(bench_filepath) or not os.path.exists(cmp_filepath): + return None, None, None + return cls.compare(bench_dir, cmp_dir, output_path, fmk) diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/dataset_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/dataset_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..96ff4809f81b8db20bc5bb26ecbf1d2e8f6e874b --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/dataset_checker.py @@ -0,0 +1,138 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +import pandas as pd +from msprobe.core.common.file_utils import create_file_in_zip, load_json +from msprobe.core.config_check.checkers.base_checker import BaseChecker +from msprobe.core.config_check.config_checker import register_checker_item, register_pre_forward_fun_list +from msprobe.core.config_check.utils.utils import config_checking_print, get_tensor_features +from msprobe.core.common.decorator import recursion_depth_decorator +from msprobe.core.common.framework_adapter import FmkAdp + + +@recursion_depth_decorator("config_check: process_obj") +def process_obj(obj): + if FmkAdp.is_tensor(obj): + return get_tensor_features(obj) + elif isinstance(obj, (tuple, list)): + return {i: process_obj(x) for i, x in enumerate(obj)} + elif isinstance(obj, dict): + return {k: process_obj(v) for k, v in obj.items()} + else: + return "" + + +def parse_args_and_kargs(args, kwargs): + processed_args = process_obj(args) + processed_kargs = process_obj(kwargs) + + return { + 'args': processed_args, + 'kwargs': processed_kargs + } + + +@recursion_depth_decorator("config_check: compare_dataset_dicts") +def compare_dataset_dicts(dict1, dict2, tag=''): + results = [] + # 处理 dict1 中的键 + for key in dict1: + new_tag = f"{tag}.{key}" if tag else key + if key not in dict2: + result = {'tag': new_tag, 'equal': False, 'status': 'delete'} + results.append(result) + continue + value1 = dict1[key] + value2 = dict2[key] + if not isinstance(value1, dict): + continue + if set(value1.keys()) == {'max', 'min', 'mean', 'norm'}: + equal = value1 == value2 + relative_diffs = { + f"{k}_relative_diff": (abs(value1[k] - value2[k]) / value1[k]) if value1[k] != 0 else None + for k in ['max', 'min', 'mean', 'norm'] + } + result = {'tag': new_tag, 'equal': equal, 'status': 'unchanged'} + result.update(relative_diffs) + results.append(result) + else: + results.extend(compare_dataset_dicts(value1, value2, new_tag)) + # 处理 dict2 中独有的键 + for key in dict2: + if key not in dict1: + new_tag = f"{tag}.{key}" if tag else key + result = {'tag': new_tag, 'equal': False, 'status': 'added'} + results.append(result) + return results + + +def compare_dataset(bench_dir, cmp_dir): + all_results = [] + for step in os.listdir(bench_dir): + step_path_bench = os.path.join(bench_dir, step) + if not os.path.isdir(step_path_bench): + continue + step_path_cmp = os.path.join(cmp_dir, step) + for rank in os.listdir(step_path_bench): + rank_path_bench = os.path.join(step_path_bench, rank, 'dataset.json') + rank_path_cmp = os.path.join(step_path_cmp, rank, 'dataset.json') + if not os.path.isfile(rank_path_bench) or not os.path.isfile(rank_path_cmp): + continue + + dict1 = load_json(rank_path_bench) + dict2 = load_json(rank_path_cmp) + results = compare_dataset_dicts(dict1, dict2) + for result in results: + result['step'] = int(step.replace("step", "")) + result['rank'] = int(rank.replace("rank", "")) + all_results.extend(results) + + df = pd.DataFrame(all_results, columns=DatasetChecker.result_header) + df = df.sort_values(by=['step', 'rank'], ascending=[True, True]) + return df + + +@register_checker_item("dataset") +class DatasetChecker(BaseChecker): + input_needed = "model" + multi_rank = True + + target_name_in_zip = "dataset" + result_header = ['step', 'rank', 'tag', 'equal', 'max_relative_diff', + 'min_relative_diff', 'mean_relative_diff', 'norm_relative_diff'] + + @staticmethod + def pack(pack_input): + output_zip_path = pack_input.output_zip_path + + def collect_input(model, args, kwargs, step): + features = parse_args_and_kargs(args, kwargs) + dataset_filepath = os.path.join(DatasetChecker.target_name_in_zip, + f"step{step}", f"rank{FmkAdp.get_rank_id()}", "dataset.json") + create_file_in_zip(output_zip_path, dataset_filepath, json.dumps(features, indent=4)) + config_checking_print(f"add first dataset input features to zip") + + register_pre_forward_fun_list(collect_input) + + @staticmethod + def compare(bench_dir, cmp_dir, output_path, fmk): + bench_dataset_pack_path = os.path.join(bench_dir, DatasetChecker.target_name_in_zip) + cmp_dataset_pack_path = os.path.join(cmp_dir, DatasetChecker.target_name_in_zip) + + df = compare_dataset(bench_dataset_pack_path, cmp_dataset_pack_path) + pass_check = False not in df['equal'].values + return DatasetChecker.target_name_in_zip, pass_check, df diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/env_args_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/env_args_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..d4f72a6b26850322aa5c7685745cfe5b54bdb8a1 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/env_args_checker.py @@ -0,0 +1,96 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json + +import pandas as pd + +from msprobe.core.common.file_utils import load_json, load_yaml, create_file_with_content, create_file_in_zip +from msprobe.core.config_check.checkers.base_checker import BaseChecker +from msprobe.core.config_check.config_checker import register_checker_item +from msprobe.core.config_check.utils.utils import config_checking_print +from msprobe.core.common.const import Const + + +dirpath = os.path.dirname(__file__) +env_yaml_path = os.path.join(dirpath, "../resource/env.yaml") + + +def collect_env_data(): + result = {} + for key, value in os.environ.items(): + result[key] = value + return result + + +def get_device_type(env_json): + for key in env_json.keys(): + if Const.ASCEND in key: + return Const.NPU_LOWERCASE + return Const.GPU_LOWERCASE + + +def compare_env_data(npu_path, bench_path): + necessary_env = load_yaml(env_yaml_path) + cmp_data = load_json(npu_path) + cmp_type = get_device_type(cmp_data) + bench_data = load_json(bench_path) + bench_type = get_device_type(bench_data) + data = [] + for _, value in necessary_env.items(): + cmp_env = value.get(cmp_type) + bench_env = value.get(bench_type) + if not bench_env and not cmp_env: + continue + elif cmp_env: + cmp_env_name = cmp_env["name"] + cmp_value = cmp_data.get(cmp_env_name, value[cmp_type]["default_value"]) + if not bench_env: + data.append(["only cmp has this env", cmp_env["name"], "", cmp_value, "warning"]) + continue + bench_env_name = bench_env["name"] + bench_value = bench_data.get(bench_env_name, value[bench_type]["default_value"]) + if cmp_value != bench_value: + data.append([bench_env_name, cmp_env_name, bench_value, cmp_value, "error"]) + else: + bench_env_name = bench_env["name"] + bench_value = bench_data.get(bench_env_name) if bench_data.get(bench_env_name) else value[bench_type][ + "default_value"] + data.append([bench_env_name, "only bench has this env", bench_value, "", "warning"]) + df = pd.DataFrame(data, columns=EnvArgsChecker.result_header) + return df + + +@register_checker_item("env") +class EnvArgsChecker(BaseChecker): + + target_name_in_zip = "env" + result_header = ["bench_env_name", "cmp_env_name", "bench_value", "cmp_value", "level"] + + @staticmethod + def pack(pack_input): + output_zip_path = pack_input.output_zip_path + env_args_dict = collect_env_data() + create_file_in_zip(output_zip_path, EnvArgsChecker.target_name_in_zip, json.dumps(env_args_dict, indent=4)) + config_checking_print(f"add env args to zip") + + @staticmethod + def compare(bench_dir, cmp_dir, output_path, fmk): + bench_env_data = os.path.join(bench_dir, EnvArgsChecker.target_name_in_zip) + cmp_env_data = os.path.join(cmp_dir, EnvArgsChecker.target_name_in_zip) + df = compare_env_data(bench_env_data, cmp_env_data) + pass_check = "error" not in df['level'].values + return EnvArgsChecker.target_name_in_zip, pass_check, df diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/hyperparameter_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/hyperparameter_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..774abef4877786268bf700bbb695586800ef64d0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/hyperparameter_checker.py @@ -0,0 +1,170 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +from difflib import SequenceMatcher + +from typing import Union, List, Dict, Any +import pandas as pd + +from msprobe.core.config_check.checkers.base_checker import BaseChecker +from msprobe.core.config_check.config_checker import register_checker_item +from msprobe.core.config_check.utils.utils import compare_dict, config_checking_print, update_dict +from msprobe.core.config_check.utils.hyperparameter_parser import ParserFactory +from msprobe.core.common.file_utils import (os_walk_for_files, create_file_in_zip, load_json, create_file_with_list, + FileOpen, load_yaml) +from msprobe.core.common.const import Const + + +dirpath = os.path.dirname(__file__) +hyperparameters_path = os.path.join(dirpath, "../resource/hyperparameter.yaml") +parameter_name_mapping = load_yaml(os.path.realpath(hyperparameters_path)) +hyperparameters_dict = {} + + +@register_checker_item("hyperparameter") +class HyperparameterChecker(BaseChecker): + target_name_in_zip = "hyperparameters" + result_header = ["file_name", "bench_para", "cmp_para", "bench_value", "cmp_value", "matched_with", "level"] + hyperparameters_file_list = ["hyperparameters_static.json", "hyperparameters_dynamic.json"] + + @staticmethod + def pack(pack_input): + shell_path = pack_input.shell_path + output_zip_path = pack_input.output_zip_path + + if shell_path: + if not isinstance(shell_path, list): + raise TypeError("shell_path should be a list of file paths.") + + hyperparameters = {} + parser_factory = ParserFactory() + for script_path in shell_path: + if os.path.isfile(script_path): + parser = parser_factory.get_parser(os.path.splitext(script_path)[1]) + update_dict(hyperparameters, parser.run(os.path.realpath(script_path))) + else: + config_checking_print(f"Warning: Script path {script_path} is not a file.") + if hyperparameters: + create_file_in_zip(output_zip_path, + os.path.join(HyperparameterChecker.target_name_in_zip, + HyperparameterChecker.hyperparameters_file_list[0]), + json.dumps(hyperparameters, indent=4)) + config_checking_print(f"add static hyperparameters args to zip") + else: + config_checking_print(f"Warning: Failed to extract hyperparameters from script {shell_path}") + if hyperparameters_dict: + create_file_in_zip(output_zip_path, + os.path.join(HyperparameterChecker.target_name_in_zip, + HyperparameterChecker.hyperparameters_file_list[1]), + json.dumps(vars(hyperparameters_dict), default=lambda x: None, indent=4)) + config_checking_print(f"add dynamic hyperparameters args to zip") + + @staticmethod + def compare(bench_dir, cmp_dir, output_path, fmk): + all_diffs = [] + for file_name in HyperparameterChecker.hyperparameters_file_list: + bench_model_dir = os.path.join(bench_dir, HyperparameterChecker.target_name_in_zip, file_name) + cmp_model_dir = os.path.join(cmp_dir, HyperparameterChecker.target_name_in_zip, file_name) + if os.path.isfile(bench_model_dir) and os.path.isfile(cmp_model_dir): + bench_hyperparameters = load_json(bench_model_dir) + cmp_hyperparameters = load_json(cmp_model_dir) + all_diffs.extend( + HyperparameterChecker.compare_param(bench_hyperparameters, cmp_hyperparameters, file_name)) + df = pd.DataFrame(all_diffs, columns=HyperparameterChecker.result_header) + pass_check = "error" not in df["level"].values + return HyperparameterChecker.target_name_in_zip, pass_check, df + + @staticmethod + def compare_param(bench_params, cmp_params, file_name): + all_diffs = [] + bench_param_names = bench_params.keys() + for bench_param_name in bench_param_names: + matched_cmp_param_name, matched_with = HyperparameterChecker._fuzzy_match_parameter(bench_param_name, + cmp_params) + bench_param_value = bench_params[bench_param_name] + if matched_cmp_param_name: + cmp_param_value = cmp_params[matched_cmp_param_name] + if bench_param_value != cmp_param_value: + all_diffs.append( + [file_name, bench_param_name, matched_cmp_param_name, bench_param_value, cmp_param_value, + matched_with, "error"]) + del cmp_params[matched_cmp_param_name] + else: + all_diffs.append( + [file_name, bench_param_name, "Only in benchmark", bench_param_value, "", "", "warning"]) + for cmp_param_name, cmp_param_value in cmp_params.items(): + all_diffs.append([file_name, "Only in comparison", cmp_param_name, "", cmp_param_value, "", "warning"]) + all_diffs.sort() + return all_diffs + + @staticmethod + def apply_patches(fmk): + try: + from megatron import training + + def collect_hyperparameter_wrapper(func): + def wrapper(*args, **kwargs): + global hyperparameters_dict + result = func(*args, **kwargs) + if not hyperparameters_dict: + hyperparameters_dict = result + return result + return wrapper + training.get_args = collect_hyperparameter_wrapper(training.get_args) + except ImportError: + config_checking_print("No megatron find.") + except Exception as e: + config_checking_print(f"Patch megatron method failed, detail:{str(e)}") + + @staticmethod + def _fuzzy_match_parameter(param_name: str, available_params: Dict[str, Any]): + """ + Fuzzy matches a parameter name against available parameter names using predefined + mappings and string similarity. + """ + if param_name in available_params: + return param_name, Const.MATCH_MODE_NAME + + canonical_name = None + for standard_name, aliases in parameter_name_mapping.items(): + if param_name == standard_name or param_name in aliases: + canonical_name = standard_name + break + + if canonical_name: + if canonical_name in available_params: + return canonical_name, Const.MATCH_MODE_MAPPING + for alias in parameter_name_mapping[canonical_name]: + if alias in available_params: + config_checking_print( + f"Matched '{param_name}' to alias '{alias}' via canonical name '{canonical_name}'") + return alias, Const.MATCH_MODE_MAPPING + + best_match_name = None + best_match_ratio = 0.8 + for available_param_name in available_params: + ratio = SequenceMatcher(None, param_name.lower(), available_param_name.lower()).ratio() + if ratio > best_match_ratio: + best_match_ratio = ratio + best_match_name = available_param_name + + if best_match_name: + config_checking_print( + f"Fuzzy matched parameter '{param_name}' to '{best_match_name}' (similarity: {best_match_ratio:.2f})") + return best_match_name, f"{Const.MATCH_MODE_SIMILARITY}:{best_match_ratio}" + + return None, None diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/pip_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/pip_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..a35bc3e00cd5bf5ed6601ce9983bad390f4b989f --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/pip_checker.py @@ -0,0 +1,90 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pandas as pd +try: + import importlib.metadata as metadata +except ImportError: + import importlib_metadata as metadata + +from msprobe.core.common.file_utils import load_yaml, create_file_in_zip +from msprobe.core.config_check.checkers.base_checker import BaseChecker +from msprobe.core.config_check.config_checker import register_checker_item +from msprobe.core.config_check.utils.utils import config_checking_print +from msprobe.core.common.file_utils import FileOpen, save_excel + +dirpath = os.path.dirname(__file__) +depend_path = os.path.join(dirpath, "../resource/dependency.yaml") + + +def load_pip_txt(file_path): + output_dir = {} + with FileOpen(file_path, 'r', encoding='utf-8') as file: + lines = file.readlines() + for line in lines: + info_list = line.strip().split("=") + output_dir[info_list[0]] = "" if len(info_list) != 2 else info_list[1] + return output_dir + + +def collect_pip_data(): + result = "" + packages = metadata.distributions() + for pkg in packages: + if pkg.metadata: + result += f"{pkg.metadata.get('Name')}={pkg.version}\n" + return result + + +def compare_pip_data(bench_pip_path, cmp_pip_path, fmk): + necessary_dependency = load_yaml(depend_path)["dependency"] + necessary_dependency.append(fmk) + bench_data = load_pip_txt(bench_pip_path) + cmp_data = load_pip_txt(cmp_pip_path) + data = [] + for package in necessary_dependency: + bench_version = bench_data.get(package) + cmp_version = cmp_data.get(package) + + if bench_version != cmp_version: + data.append([package, bench_version if bench_version else 'None', + cmp_version if cmp_version else 'None', + "error"]) + + df = pd.DataFrame(data, columns=PipPackageChecker.result_header) + return df + + +@register_checker_item("pip") +class PipPackageChecker(BaseChecker): + + target_name_in_zip = "pip" + result_header = ['package', 'bench version', 'cmp version', 'level'] + + @staticmethod + def pack(pack_input): + output_zip_path = pack_input.output_zip_path + pip_data = collect_pip_data() + create_file_in_zip(output_zip_path, PipPackageChecker.target_name_in_zip, pip_data) + config_checking_print(f"add pip info to zip") + + @staticmethod + def compare(bench_dir, cmp_dir, output_path, fmk): + bench_pip_path = os.path.join(bench_dir, PipPackageChecker.target_name_in_zip) + cmp_pip_path = os.path.join(cmp_dir, PipPackageChecker.target_name_in_zip) + df = compare_pip_data(bench_pip_path, cmp_pip_path, fmk) + pass_check = "error" not in df['level'].values + return PipPackageChecker.target_name_in_zip, pass_check, df diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/random_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/random_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..1d1d0a7e79feb63116dc40c139b74c9d5778a8f0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/random_checker.py @@ -0,0 +1,208 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import random +from functools import wraps +from typing import Callable +import inspect +import os +import json +from collections import defaultdict + +import numpy as np +import pandas as pd +from msprobe.core.config_check.config_checker import register_checker_item, register_pre_forward_fun_list +from msprobe.core.common.file_utils import create_file_in_zip, load_json +from msprobe.core.config_check.checkers.base_checker import BaseChecker +from msprobe.core.config_check.utils.utils import config_checking_print +from msprobe.core.common.framework_adapter import FmkAdp +from msprobe.core.common.const import Const + + +random_log_dict = defaultdict(dict) + + +def load_json_files(directory): + json_data = {} + for file in os.listdir(directory): + file_path = os.path.join(directory, file) + if file.startswith('rank') and file.endswith('.json'): + json_data.update(load_json(file_path)) + return json_data + + +def get_file_and_line(position): + parts = position.rsplit(':', 1) + if len(parts) == 2: + file_name = os.path.basename(parts[0]) + line_num = parts[1] + return f"{file_name}:{line_num}" + return position + + +def compare_json_files(bench_data, cmp_data): + results = [] + for op in set(bench_data) | set(cmp_data): + bench_records = bench_data.get(op, {}) + cmp_records = cmp_data.get(op, {}) + all_positions = set() + for position in set(bench_records) | set(cmp_records): + all_positions.add(get_file_and_line(position)) + + for position in all_positions: + bench_count = 0 + cmp_count = 0 + for original_position, count in bench_records.items(): + if get_file_and_line(original_position) == position: + bench_count += count + for original_position, count in cmp_records.items(): + if get_file_and_line(original_position) == position: + cmp_count += count + results.append([op, position, bench_count == cmp_count, bench_count, cmp_count]) + return results + + +def compare_random(bench_dir='bench', cmp_dir='cmp'): + bench_data = load_json_files(bench_dir) + cmp_data = load_json_files(cmp_dir) + results = compare_json_files(bench_data, cmp_data) + df = pd.DataFrame(results, columns=RandomChecker.result_header) + return df + + +def track_random_call(func: Callable, name: str): + @wraps(func) + def wrapper(*args, **kwargs): + frame = inspect.currentframe() + caller_frame = frame.f_back + caller_info = inspect.getframeinfo(caller_frame) + location = f"{os.path.abspath(caller_info.filename)}:{caller_info.lineno}" + + global random_log_dict + random_log_dict.setdefault(name, {}) + random_log_dict[name][location] = random_log_dict[name].get(location, 0) + 1 + + try: + result = func(*args, **kwargs) + return result + except Exception as e: + raise e + finally: + del frame, caller_frame + + return wrapper + + +def torch_patchs(): + import torch + torch_patches = { + 'rand': torch.rand, + 'randint': torch.randint, + 'randn': torch.randn, + 'rand_like': torch.rand_like, + 'randint_like': torch.randint_like, + 'randn_like': torch.randn_like, + 'manual_seed': torch.manual_seed + } + for name, func in torch_patches.items(): + setattr(torch, name, track_random_call(func, f"torch.{name}")) + + tensor_patches = { + 'exponential_': torch.Tensor.exponential_, + 'geometric_': torch.Tensor.geometric_, + 'log_normal_': torch.Tensor.log_normal_, + 'cauchy_': torch.Tensor.cauchy_ + } + for name, func in tensor_patches.items(): + setattr(torch.Tensor, name, track_random_call(func, f"torch.Tensor.{name}")) + + +def mindspore_patchs(): + import mindspore + + mindspore_ops_patches = { + 'rand': mindspore.ops.uniform, + 'randint': mindspore.ops.randint, + 'randn': mindspore.ops.normal + } + for name, func in mindspore_ops_patches.items(): + setattr(mindspore.ops, name, track_random_call(func, f"mindspore.ops.{name}")) + + mindspore_patches = { + 'manual_seed': mindspore.set_seed + } + for name, func in mindspore_patches.items(): + setattr(mindspore, name, track_random_call(func, f"mindspore.{name}")) + + +@register_checker_item("random") +class RandomChecker(BaseChecker): + input_needed = None + + target_name_in_zip = "random" + result_header = ['op', 'position', 'equal', 'bench_count', 'cmp_count'] + write_once = False + + @staticmethod + def pack(pack_input): + output_zip_path = pack_input.output_zip_path + + def collect_input(model, args, kwargs, step): + if RandomChecker.write_once: + return + + random_log_filepath = os.path.join(RandomChecker.target_name_in_zip, f"rank{FmkAdp.get_rank_id()}.json") + create_file_in_zip(output_zip_path, random_log_filepath, json.dumps(random_log_dict, indent=4)) + config_checking_print(f"add first random_log input features to zip") + RandomChecker.write_once = True + + register_pre_forward_fun_list(collect_input) + + @staticmethod + def compare(bench_dir, cmp_dir, output_path, fmk): + bench_random_log_pack_path = os.path.join(bench_dir, RandomChecker.target_name_in_zip) + cmp_random_log_pack_path = os.path.join(cmp_dir, RandomChecker.target_name_in_zip) + + df = compare_random(bench_random_log_pack_path, cmp_random_log_pack_path) + pass_check = False not in df['equal'].values + return RandomChecker.target_name_in_zip, pass_check, df + + @staticmethod + def apply_patches(fmk=Const.PT_FRAMEWORK): + random_patches = { + 'random': random.random, + 'randint': random.randint, + 'uniform': random.uniform, + 'choice': random.choice + } + for name, func in random_patches.items(): + setattr(random, name, track_random_call(func, f"random.{name}")) + + np_random_patches = { + 'rand': np.random.rand, + 'randint': np.random.randint, + 'choice': np.random.choice, + 'normal': np.random.normal + } + for name, func in np_random_patches.items(): + setattr(np.random, name, track_random_call(func, f"np.random.{name}")) + + if fmk == Const.PT_FRAMEWORK: + torch_patchs() + elif fmk == Const.MS_FRAMEWORK: + mindspore_patchs() + else: + raise Exception(f"apply patches framework error, not in {FmkAdp.supported_fmk}") diff --git a/debug/accuracy_tools/msprobe/core/config_check/checkers/weights_checker.py b/debug/accuracy_tools/msprobe/core/config_check/checkers/weights_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..876e68ef029993918704003d6369ce06d2c84bd3 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/checkers/weights_checker.py @@ -0,0 +1,144 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +import pandas as pd + +from msprobe.core.common.file_utils import create_file_in_zip, os_walk_for_files, load_json +from msprobe.core.config_check.checkers.base_checker import BaseChecker +from msprobe.core.config_check.config_checker import register_checker_item, register_pre_forward_fun_list +from msprobe.core.config_check.utils.utils import config_checking_print, get_tensor_features +from msprobe.core.common.framework_adapter import FmkAdp + + +def collect_weights_data(model): + weights_data = {} + for name, param in FmkAdp.named_parameters(model): + if param.dtype != FmkAdp.dtype("float32"): + param = param.float() + weights_data[name] = get_tensor_features(param) + return weights_data + + +def compare_weight_file(bench_file, cmp_file): + bench_data = load_json(bench_file) + cmp_data = load_json(cmp_file) + + results = [] + for weight_name in set(bench_data.keys()) | set(cmp_data.keys()): + result = { + "weight_name": weight_name, + "equal": None, + "max_relative_diff": None, + "min_relative_diff": None, + "mean_relative_diff": None, + "norm_relative_diff": None + } + + if weight_name not in bench_data: + result["equal"] = "only cmp have" + results.append(result) + continue + + if weight_name not in cmp_data: + result["equal"] = "only bench have" + results.append(result) + continue + + bench_vals = bench_data[weight_name] + cmp_vals = cmp_data[weight_name] + keys = ["max", "min", "mean", "norm"] + equal = all([bench_vals[k] == cmp_vals[k] for k in keys]) + result["equal"] = equal + + for key in keys: + diff_key = f"{key}_relative_diff" + result[diff_key] = (abs(bench_vals[key] - cmp_vals[key]) / bench_vals[key]) \ + if bench_vals[key] != 0 else None + + results.append(result) + + return results + + +def compare_weight(bench_dir, cmp_dir): + all_results = [] + bench_files_info = os_walk_for_files(bench_dir, 10) + for info in bench_files_info: + if not info["file"].endswith('.json'): + continue + bench_file = os.path.join(info["root"], info["file"]) + relative_path = os.path.relpath(info["root"], bench_dir) + cmp_root = os.path.join(cmp_dir, relative_path) + cmp_file = os.path.join(cmp_root, info["file"]) + + step = int(relative_path.split(os.sep)[0].replace("step", "")) + rank = int(relative_path.split(os.sep)[1].replace("rank", "")) + + if not os.path.exists(cmp_file): + bench_data = load_json(bench_file) + for weight_name in bench_data.keys(): + result = { + "step": step, + "rank": rank, + "weight_name": weight_name, + "equal": "only bench have", + "max_relative_diff": None, + "min_relative_diff": None, + "mean_relative_diff": None, + "norm_relative_diff": None + } + all_results.append(result) + else: + results = compare_weight_file(bench_file, cmp_file) + for res in results: + res["step"] = step + res["rank"] = rank + all_results.append(res) + + df = pd.DataFrame(all_results, columns=WeightsChecker.result_header) + df = df.sort_values(by=['step', 'rank'], ascending=[True, True]) + return df + + +@register_checker_item("weights") +class WeightsChecker(BaseChecker): + input_needed = "model" + multi_rank = True + + target_name_in_zip = "weights" + result_header = ["step", "rank", "weight_name", "equal", "max_relative_diff", + "min_relative_diff", "mean_relative_diff", "norm_relative_diff"] + + @staticmethod + def pack(pack_input): + output_zip_path = pack_input.output_zip_path + + def collect_weights(model, args, kwargs, step): + weights_data_dict = collect_weights_data(model) + weights_data_filepath = os.path.join(WeightsChecker.target_name_in_zip, + f"step{step}", f"rank{FmkAdp.get_rank_id()}", "weight.json") + create_file_in_zip(output_zip_path, weights_data_filepath, json.dumps(weights_data_dict, indent=4)) + config_checking_print(f"add weights info to zip") + register_pre_forward_fun_list(collect_weights) + + @staticmethod + def compare(bench_dir, cmp_dir, output_path, fmk): + bench_weight_pack_path = os.path.join(bench_dir, WeightsChecker.target_name_in_zip) + cmp_weight_pack_path = os.path.join(cmp_dir, WeightsChecker.target_name_in_zip) + df = compare_weight(bench_weight_pack_path, cmp_weight_pack_path) + pass_check = False not in df['equal'].values + return WeightsChecker.target_name_in_zip, pass_check, df diff --git a/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/ckpt_comparator.py b/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/ckpt_comparator.py new file mode 100644 index 0000000000000000000000000000000000000000..3c088c249a3088a9768accb0b6c2a4d429a6fab0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/ckpt_comparator.py @@ -0,0 +1,74 @@ +# Copyright (c) 2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict +from tqdm import tqdm + +from msprobe.core.common.file_utils import save_json, check_path_before_create, check_path_not_exists +from msprobe.core.common.log import logger +from msprobe.core.config_check.ckpt_compare.megatron_loader import load_megatron_weights +from msprobe.core.config_check.ckpt_compare.metrics import METRIC_FUNC + + + +def compare_checkpoints(ckpt_path1, ckpt_path2, output_path) -> Dict: + """Compare weights between two checkpoints using cosine similarity and L2 distance. + + Args: + ckpt_path1 (str): Path to first checkpoint directory + ckpt_path2 (str): Path to second checkpoint directory + output_path (str): Path to save comparison results JSON file + + Returns: + Dict: Dictionary containing comparison metrics for each parameter. The dictionary has the following structure: + { + "param_name": { + "cosine_similarity": float, # Cosine similarity between parameter tensors + "l2_distance": float, # L2 distance between parameter tensors + "shape": List[int] # Shape of the parameter tensors + }, + ... + } + """ + + # Load both checkpoints + check_path_before_create(output_path) + check_path_not_exists(output_path) + weights1 = load_megatron_weights(ckpt_path1) + weights2 = load_megatron_weights(ckpt_path2) + + # Initialize results dictionary + results = {} + + # Compare weights with matching keys + common = set(weights1) & set(weights2) + logger.warning(f'Parameters not in ckpt2: {set(weights1) - set(weights2)}') + logger.warning(f'Parameters not in ckpt1: {set(weights2) - set(weights1)}') + for key in tqdm(common): + tensor1 = weights1[key] + tensor2 = weights2[key] + + results[key] = {} + for metric, func in METRIC_FUNC.items(): + try: + results[key][metric] = func(tensor1, tensor2) + except Exception as e: + results[key][metric] = 'error' + logger.warning(f'Error when calculate {metric} for reason: {e}') + + # Write results to JSON file + save_json(output_path, results, indent=4) + logger.info(f"Comparison results written to {output_path}") + return results diff --git a/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/megatron_loader.py b/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/megatron_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..af1c5518aacfa5d02f21633d1bb162e41f979917 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/megatron_loader.py @@ -0,0 +1,302 @@ +# Copyright (c) 2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from collections import defaultdict +from typing import Dict +import numpy as np +from msprobe.core.common.log import logger +from msprobe.core.common.decorator import recursion_depth_decorator +from msprobe.core.common.const import Const +from msprobe.core.common.file_utils import FileOpen, load_yaml +from msprobe.core.common.framework_adapter import FmkAdp + +# both weights and bias are partitioned in column parallel +COLUMN_PARALLEL_PARAMS = ['linear_qkv', 'linear_fc1', 'word_embeddings.weight', 'output_layer.weight'] +# only weights are partitioned in column parallel +ROW_PARALLEL_PARAMS = ['linear_fc2.weight', 'linear_proj.weight'] +ARGS = 'args' +LAYER_IDX_PATTERN = re.compile('layers\.(\d+)\.') +EXPERT_IDX_PATTERN = re.compile('experts\.(\d+)\.') +ITER_DIR_PATTERN = re.compile('iter_([\d]{7})') + + +@recursion_depth_decorator('') +def _get_parameter(weights, prefix=''): + for k, v in weights.items(): + name = Const.SEP.join([prefix, k]).strip(Const.SEP) + if isinstance(v, dict): + yield from _get_parameter(v, prefix=name) + elif FmkAdp.is_tensor(v): + yield name, FmkAdp.asnumpy(v) + + +def _map_to_mcore_local_names(param_name: str) -> str: + """Map parameter names to mcore + local transformer implementation names.""" + mcore_local_map = load_yaml(os.path.join(os.path.dirname(__file__), 'name_mapping.yaml')) + for other_name, mcore_local_name in mcore_local_map.items(): + param_name = param_name.replace(other_name, mcore_local_name) + + return param_name + + +def _parse_real_layer_idx(param_name, num_layers_per_stage, pp_size, pp_rank): + """Map local (virtual) pipeline stage layer index to global layer index. + + For virtual pipeline parallel, each pipeline stage is further divided into virtual stages. + The global layer index needs to account for both pipeline stage and virtual stage. + + Args: + param_name (str): Parameter name containing layer index: layers.x./ + num_layers_per_stage (int): Number of layers per pipeline stage + pp_size (int): Pipeline parallel size + + Returns: + int: Global layer index accounting for both pipeline and virtual pipeline stages + """ + # Extract local layer index from parameter name + layer_match = re.search(LAYER_IDX_PATTERN, param_name) + param_name, vpp_stage = param_name.split(Const.SCOPE_SEPARATOR) + if not layer_match: + return param_name + + local_layer_idx = int(layer_match.group(1)) + vpp_stage = int(vpp_stage) + + # Calculate global layer index based on pipeline stage and virtual stage + real_layer_idx = local_layer_idx + (pp_size * vpp_stage + pp_rank) * num_layers_per_stage + + return param_name.replace(f'layers.{local_layer_idx}', f'layers.{real_layer_idx}') + + +def _parse_real_expert_idx(param_name, num_experts_per_rank, exp_rank): + """Map local expert index to global expert index. TODO: shared expert + + For expert parallel, experts are distributed across ranks. This function maps + the local expert index on a rank to its global index across all ranks. + + Args: + param_name (str): Parameter name containing local expert index + num_experts_per_rank (int): Number of experts on each rank + exp_rank (int): Expert parallel rank + + Returns: + str: Parameter name with local expert index replaced by global expert index + """ + # Extract local layer index from parameter name + expert_match = re.search(EXPERT_IDX_PATTERN, param_name) + if not expert_match: + return param_name + + local_expert_idx = int(expert_match.group(1)) + # Calculate global layer index based on pipeline stage and virtual stage + real_experts_idx = local_expert_idx + exp_rank * num_experts_per_rank + + return param_name.replace(f'experts.{local_expert_idx}', f'experts.{real_experts_idx}') + + +def _consolidate_tp_weights(weights: Dict) -> Dict: + """Consolidate weights from different tensor parallel ranks into combined tensors. + + Args: + weights: Dictionary of weights with rank information in keys + + Returns: + Dict: Consolidated weights without rank information + """ + consolidated = {} + for key, tensors in weights.items(): + if any([name in key for name in COLUMN_PARALLEL_PARAMS]): + # Column parallel - concatenate along input dimension (dim 0) + combined = np.concatenate(tensors, axis=0) + elif any([name in key for name in ROW_PARALLEL_PARAMS]): + # Row parallel - concatenate along output dimension (dim 1) + combined = np.concatenate(tensors, axis=1) + else: + # For other params, verify identical and use first + if not all(np.allclose(tensors[0], t) for t in tensors[1:]): + logger.warning(f"Inconsistent values for {key} across TP ranks") + combined = tensors[0] + + consolidated[key] = combined + return consolidated + + +def _parse_num_layers_per_stage(tp_partition): + match = [re.findall(LAYER_IDX_PATTERN, key) for key in tp_partition.keys()] + layer_idx = [int(i[0]) for i in match if i] + num_layers_per_pipeline_stage = max(layer_idx) + 1 + + return num_layers_per_pipeline_stage + + +def parse_parallel_size(checkpoint_dir: str): + """Parse tensor, pipeline and expert parallel sizes from checkpoint filenames. + + Args: + checkpoint_dir (str): Directory containing checkpoint files + + Returns: + Namespace + """ + # Find all rank directories + rank_dirs = [d for d in os.listdir(checkpoint_dir) if d.startswith('mp_rank_')] + + if not rank_dirs: + raise ValueError(f"No checkpoint rank directories found in {checkpoint_dir}") + + ckpt = FmkAdp.load_checkpoint( + os.path.join(checkpoint_dir, rank_dirs[0], 'model_optim_rng.pt'), + to_cpu=True, + weights_only=False) + args = ckpt[ARGS] + return ( + args.tensor_model_parallel_size, + args.pipeline_model_parallel_size, + args.expert_model_parallel_size, + args.num_experts + ) + + +def parse_iteration(checkpoint_path: str) -> Dict: + """ + Parse the checkpoint iteration directory from a given checkpoint path. + + If the path is a top-level checkpoint directory, this function reads the + 'latest_checkpointed_iteration.txt' file to determine the latest iteration. + If the path is already an iteration directory (e.g., 'iter_0000005'), it extracts + the iteration number from the path. + + Args: + checkpoint_path (str): Path to the checkpoint directory or iteration directory. + + Returns: + str: The full path to the checkpoint directory for the determined iteration. + + Raises: + ValueError: If the checkpoint directory for the determined iteration does not exist. + """ + iteration = None + tracker_file = os.path.join(checkpoint_path, "latest_checkpointed_iteration.txt") + if os.path.exists(tracker_file): + with FileOpen(tracker_file, 'r') as f: + latest_iteration = f.read().strip() + if latest_iteration != 'release': + try: + iteration = int(latest_iteration) + except Exception: + logger.warning( + f"The latest_checkpointed_iteration is supposed to be `release` or an int. \ + But {latest_iteration} is found." + ) + checkpoint_path = os.path.join(checkpoint_path, f'iter_{iteration:07d}') + else: + match = re.findall(ITER_DIR_PATTERN, checkpoint_path) + if match: + iteration = int(match[0]) + + # Checkpoint directory for this iteration + logger.info(f"Loaded checkpoint from iteration {iteration}") + + if not os.path.exists(checkpoint_path): + raise ValueError(f"Checkpoint directory not found: {checkpoint_path}") + + return checkpoint_path + + +def get_weights_from_state_dict(state_dict): + weights = {} + vpp_stage = 0 + if 'model' in state_dict: + model_weights = state_dict['model'] + + for key, value in _get_parameter(model_weights): + key = _map_to_mcore_local_names(key) + weights[f"{key}{Const.SCOPE_SEPARATOR}{vpp_stage}"] = value + + elif 'model0' in state_dict: + #vpp enabled + while f'model{vpp_stage}' in state_dict: + model_weights = state_dict[f'model{vpp_stage}'] + for key, value in _get_parameter(model_weights): + key = _map_to_mcore_local_names(key) + weights[f"{key}{Const.SCOPE_SEPARATOR}{vpp_stage}"] = value + vpp_stage += 1 + return weights + + +def load_megatron_weights(checkpoint_path: str) -> Dict: + """Load Megatron parallel checkpoint weights into a single dictionary. + + Args: + checkpoint_path (str): Base checkpoint directory path + + Returns: + combined_weights: Dict with weights from all ranks, keys include rank info + """ + try: + import megatron + except ModuleNotFoundError as e: + raise ModuleNotFoundError("No module named 'megatron', which is required to load a megatron ckpt") from e + + # Find latest iteration if not specified + checkpoint_path = parse_iteration(checkpoint_path) + + # Parse parallel sizes from checkpoint directory structure + tp_size, pp_size, exp_size, num_experts = parse_parallel_size(checkpoint_path) + combined_weights = {} + + # Load checkpoints from all ranks + for exp_rank in range(exp_size): + num_layers_per_pipeline_stage = 0 + for pp_rank in range(pp_size): + tp_partition = defaultdict(list) + for tp_rank in range(tp_size): + # Construct checkpoint path based on parallel ranks + if pp_size > 1: + rank_dir = f'mp_rank_{tp_rank:02d}_{pp_rank:03d}' + else: + rank_dir = f'mp_rank_{tp_rank:02d}' + + if exp_size > 1: + rank_dir = f'{rank_dir}_{exp_rank:03d}' + + ckpt_file = os.path.join(checkpoint_path, rank_dir, 'model_optim_rng.pt') + try: + state_dict = FmkAdp.load_checkpoint(ckpt_file, to_cpu=True, weights_only=False) + partition = get_weights_from_state_dict(state_dict) + for key, weight in partition.items(): + tp_partition[key].append(weight) + + except Exception as load_error: + logger.warning(f"Error loading {ckpt_file}: {load_error}") + + if not tp_partition: + raise ValueError('No state loaded.') + + if not num_layers_per_pipeline_stage: + num_layers_per_pipeline_stage = _parse_num_layers_per_stage(tp_partition) + + consolidated_weight = _consolidate_tp_weights(tp_partition) + for key, value in consolidated_weight.items(): + key = _parse_real_layer_idx(key, num_layers_per_pipeline_stage, pp_size, pp_rank) + if num_experts: + key = _parse_real_expert_idx(key, num_experts // exp_size, exp_rank) + combined_weights[key] = value + + logger.info(f"Found {len(combined_weights)} total parameters across all ranks") + + return combined_weights diff --git a/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/metrics.py b/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/metrics.py new file mode 100644 index 0000000000000000000000000000000000000000..2e9e1324b33c570033fa4fc29a6a32dff73b64de --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/metrics.py @@ -0,0 +1,83 @@ +# Copyright (c) 2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np + +from msprobe.core.common.log import logger +from msprobe.core.compare.npy_compare import CompareOps + + + +def in_different_shape(a, b): + if a.shape != b.shape: + logger.warning(f"a, b are in different shape. a: {a.shape}, b: {b.shape}") + return True + return False + + +def l2_distance(a, b): + if a is None or b is None: + return None + if in_different_shape(a, b): + return None + return np.linalg.norm(a - b).item() + + +def cos_sim(a, b): + if a is None or b is None: + return None + + if in_different_shape(a, b): + return None + if a.ndim > 0: + a = a.flatten().squeeze() + b = b.flatten().squeeze() + + num = a.dot(b) + a_norm = np.linalg.norm(a) + b_norm = np.linalg.norm(b) + + if a_norm == 0 and b_norm == 0: + return 1. + if a_norm == 0 or b_norm == 0: + logger.warning(f'One tensor norm is zero.') + return None + + sim = num / (a_norm * b_norm) + + return sim.item() + + +def numel(a, b): + n1 = a.size + n2 = b.size + if n1 != n2: + logger.warning('parameters have different number of element') + return (n1, n2) + return n1 + + +def shape(a, b): + if in_different_shape(a, b): + return [list(a.shape), list(b.shape)] + return list(a.shape) + + +METRIC_FUNC = { + 'l2': l2_distance, + 'cos': cos_sim, + 'numel': numel, + 'shape': shape + } diff --git a/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/name_mapping.yaml b/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/name_mapping.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0caecc53a73b108939435867fe1b6e614bd91812 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/ckpt_compare/name_mapping.yaml @@ -0,0 +1,12 @@ +self_attention.linear_qkv.layer_norm_: input_layernorm. +language_model.: '' +encoder: decoder +.input_norm.: .input_layernorm. +query_key_value: linear_qkv +.dense.: .linear_proj. +post_attention_norm: pre_mlp_layernorm +dense_h_to_4h: linear_fc1 +dense_4h_to_h: linear_fc2 +mlp.local_experts: mlp.experts.local_experts +final_norm: final_layernorm +word_embeddings_for_head: output_layer diff --git a/debug/accuracy_tools/msprobe/core/config_check/config_check_cli.py b/debug/accuracy_tools/msprobe/core/config_check/config_check_cli.py new file mode 100644 index 0000000000000000000000000000000000000000..cc2db192416517d6b94020441d9edc1eff95f89b --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/config_check_cli.py @@ -0,0 +1,51 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from msprobe.core.config_check.config_checker import ConfigChecker +from msprobe.core.config_check.ckpt_compare.ckpt_comparator import compare_checkpoints +from msprobe.core.common.log import logger + + +def pack(shell_path, output_path, framework): + ConfigChecker(shell_path=shell_path, output_zip_path=output_path, fmk=framework) + + +def compare(bench_zip_path, cmp_zip_path, output_path, framework): + ConfigChecker.compare(bench_zip_path, cmp_zip_path, output_path, framework) + + +def _config_checking_parser(parser): + parser.add_argument('-d', '--dump', nargs='*', help='Collect the train config into a zip file') + parser.add_argument('-c', '--compare', nargs=2, help='Compare two zip files or checkpoints') + parser.add_argument('-o', '--output', help='output path, default is current directory') + + +def _run_config_checking_command(args): + if args.dump is not None: + output_dirpath = args.output if args.output else "./config_check_pack.zip" + pack(args.dump, output_dirpath, args.framework) + elif args.compare: + if args.compare[0].endswith('zip'): + logger.info('The input paths is zip files, comparing packed config.') + output_dirpath = args.output if args.output else "./config_check_result" + compare(args.compare[0], args.compare[1], output_dirpath, args.framework) + else: + logger.info('Comparing model checkpoint.') + output_dirpath = args.output if args.output else "./ckpt_similarity.json" + compare_checkpoints(args.compare[0], args.compare[1], output_dirpath) + + else: + logger.error("The param is not correct, you need to give '-d' for dump or '-c' for compare.") + raise Exception("The param is not correct, you need to give '-d' for dump or '-c' for compare.") diff --git a/debug/accuracy_tools/msprobe/core/config_check/config_checker.py b/debug/accuracy_tools/msprobe/core/config_check/config_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..2dc908398b83d1f5c15b5dcefdcc93a4a2ef58a4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/config_checker.py @@ -0,0 +1,100 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil + +import pandas as pd + +from msprobe.core.common.file_utils import save_excel, split_zip_file_path, \ + create_directory, extract_zip +from msprobe.core.common.framework_adapter import FmkAdp +from msprobe.core.config_check.checkers.base_checker import PackInput +from msprobe.core.config_check.utils.utils import config_checking_print +from msprobe.core.common.const import Const + + +class ConfigChecker: + checkers = {} + pre_forward_fun_list = [] + result_filename = "result.xlsx" + result_header = ["filename", "pass_check"] + step = 0 + + def __init__(self, model=None, shell_path=None, output_zip_path="./config_check_pack.zip", fmk="pytorch"): + FmkAdp.set_fmk(fmk) + self.pack_input = PackInput(output_zip_path, model, shell_path) + file_path, file_name = split_zip_file_path(self.pack_input.output_zip_path) + if not os.path.exists(file_path): + create_directory(file_path) + self.pack() + + @staticmethod + def compare(bench_zip_path, cmp_zip_path, output_path, fmk=Const.PT_FRAMEWORK): + if os.path.exists(output_path): + shutil.rmtree(output_path) + bench_dir = os.path.join(output_path, "bench") + cmp_dir = os.path.join(output_path, "cmp") + extract_zip(bench_zip_path, bench_dir) + config_checking_print(f"extract zip file {bench_zip_path} to {bench_dir}") + extract_zip(cmp_zip_path, cmp_dir) + config_checking_print(f"extract zip file {cmp_zip_path} to {cmp_dir}") + + result = [] + summary_result = [] + for checker in ConfigChecker.checkers.values(): + checker_name, pass_check, df = checker.compare_ex(bench_dir, cmp_dir, output_path, fmk) + if checker_name: + summary_result.append([checker_name, pass_check]) + if df is not None: + result.append((df, checker_name)) + summary_result_df = pd.DataFrame(summary_result, columns=ConfigChecker.result_header) + result.insert(0, (summary_result_df, "summary")) + save_excel(os.path.join(output_path, ConfigChecker.result_filename), result) + config_checking_print(f"config checking result save to {os.path.realpath(output_path)}") + + @staticmethod + def apply_patches(fmk=Const.PT_FRAMEWORK): + for checker in ConfigChecker.checkers.values(): + checker.apply_patches(fmk) + + def pack(self): + config_checking_print(f"pack result zip path {os.path.realpath(self.pack_input.output_zip_path)}") + + def hook(model, args, kwargs): + for collect_func in self.pre_forward_fun_list: + collect_func(model, args, kwargs, ConfigChecker.step) + ConfigChecker.step += 1 + + if self.pack_input.model: + FmkAdp.register_forward_pre_hook(self.pack_input.model, hook, with_kwargs=True) + for checker in ConfigChecker.checkers.values(): + if checker.input_needed and not getattr(self.pack_input, checker.input_needed): + continue + if FmkAdp.is_initialized() and FmkAdp.get_rank() != 0 and not checker.multi_rank: + continue + checker.pack(self.pack_input) + + +def register_checker_item(key, cls=None): + if cls is None: + # 无参数时，返回装饰器函数 + return lambda cls: register_checker_item(key, cls) + ConfigChecker.checkers[key] = cls + return cls + + +def register_pre_forward_fun_list(func): + ConfigChecker.pre_forward_fun_list.append(func) diff --git a/debug/accuracy_tools/msprobe/pytorch/parse.py b/debug/accuracy_tools/msprobe/core/config_check/resource/dependency.yaml similarity index 87% rename from debug/accuracy_tools/msprobe/pytorch/parse.py rename to debug/accuracy_tools/msprobe/core/config_check/resource/dependency.yaml index 3dfd88f03d1b944f6943a58ce860c7de9c4a3424..02c0b565bf59b1b220f16ae17a47f5f4d5b13c1f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse.py +++ b/debug/accuracy_tools/msprobe/core/config_check/resource/dependency.yaml @@ -13,7 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from msprobe.pytorch.parse_tool import cli - -if __name__ == '__main__': - cli.parse() +dependency: + - transformers + - deepspeed + - megatron + - numpy + - datasets + - peft \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/config_check/resource/env.yaml b/debug/accuracy_tools/msprobe/core/config_check/resource/env.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87d663b9d94976c24feb88b181b3ead98905eb5a --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/resource/env.yaml @@ -0,0 +1,57 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +HCCL_DETERMINISTIC: + npu: + name: HCCL_DETERMINISTIC + default_value: False + gpu: + name: NCCL_DETERMINISTIC + default_value: False + +HCCL_ALGO: + npu: + name: HCCL_ALGO + default_value: None + gpu: + name: NCCL_ALGO + default_value: None + +HCCL_INTRA_ROCE_ENABLE: + npu: + name: HCCL_INTRA_ROCE_ENABLE + default_value: 0 + + +HCCL_INTRA_PICE_ENABLE: + npu: + name: HCCL_INTRA_ROCE_ENABLE + default_value: 1 + +ASCEND_LAUNCH_BLOCKING: + npu: + name: ASCEND_LAUNCH_BLOCKING + default_value: 0 + gpu: + name: CUDA_LAUNCH_BLOCKING + default_value: 0 + +ASCEND_RT_VISIBLE_DEVICES: + npu: + name: ASCEND_RT_VISIBLE_DEVICES + default_value: None + gpu: + name: CUDA_VISIBLE_DEVICES + default_value: None \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/config_check/resource/hyperparameter.yaml b/debug/accuracy_tools/msprobe/core/config_check/resource/hyperparameter.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cff815717fc5b668bdd5f99de1a18e0373760fe --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/resource/hyperparameter.yaml @@ -0,0 +1,21 @@ +learning_rate: + - lr + - learningrate + +batch_size: + - batch + - bs + - batch_size_per_gpu + +epochs: + - num_epochs + - max_epochs + - epoch + +weight_decay: + - wd + - weightdecay + +dropout_rate: + - dropout + - drop_rate \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/config_check/utils/hyperparameter_parser.py b/debug/accuracy_tools/msprobe/core/config_check/utils/hyperparameter_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..6cb540ee49951652b6094f80229da099cfc5afdf --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/utils/hyperparameter_parser.py @@ -0,0 +1,115 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from abc import ABC, abstractmethod + +from msprobe.core.config_check.utils.utils import config_checking_print +from msprobe.core.common.file_utils import FileOpen, load_yaml +from msprobe.core.common.const import Const, FileCheckConst + + +class Parser(ABC): + @abstractmethod + def parse(self, file_path: str) -> dict: + pass + + def run(self, file_path: str) -> dict: + """ + 统一对外调用接口 + :param file_path: 需解析的文件路径 + :return: + """ + try: + result = self.parse(file_path) + except Exception as exc: + config_checking_print(f"{self.__class__} parsing error, skip file path: {file_path}, error: {exc}") + result = {} + return result + + +class ShellParser(Parser): + def parse(self, file_path: str) -> dict: + """ + Extracts arguments from bash script used to run a model training. + """ + hyperparameters = {} + script_content_list = [] + with FileOpen(file_path, 'r') as file: + for line in file: + stripped_line = line.lstrip() + if not stripped_line.startswith('#'): + line = line.split('#')[0].rstrip() + '\n' + if line.strip(): + script_content_list.append(line) + script_content = ''.join(script_content_list) + + command_line = re.search(r'msrun\s[^|]*|torchrun\s[^|]*|python\d? -m torch.distributed.launch\s[^|]*', + script_content, + re.DOTALL) + if command_line: + command_line = command_line.group() + + blocks = re.findall(r'([a-zA-Z0-9_]{1,20}_ARGS)="(.*?)"', script_content, re.DOTALL) + block_contents = {} + for block_name, block_content in blocks: + block_content = block_content.replace('\n', ' ') + block_contents[block_name] = block_content + command_line = command_line.replace(f"${block_name}", block_content) + + matches = re.findall(r'--([\w-]+)(?:\s+([^\s\\]+))?', command_line) + for match in matches: + key, value = match + args_key = re.match(r'\$\{?(\w+)}?', value) + if args_key: + env_vars = re.findall(rf'{args_key.group(1)}=\s*(.+)', script_content) + if env_vars: + value = env_vars[-1] + hyperparameters[key] = value if value else True + + return hyperparameters + + +class YamlParser(Parser): + hyperparameters = {} + + def parse(self, file_path: str) -> dict: + ori_hyper = load_yaml(file_path) + self.recursive_parse_parameters(ori_hyper, "") + return self.hyperparameters + + def recursive_parse_parameters(self, parameters, prefix): + if isinstance(parameters, dict): + for key, value in parameters.items(): + new_prefix = prefix + Const.SEP + key if prefix else key + self.recursive_parse_parameters(value, new_prefix) + elif isinstance(parameters, list): + for value in parameters: + self.recursive_parse_parameters(value, prefix) + elif isinstance(parameters, (int, str, bool)): + self.hyperparameters.update({prefix: parameters}) + + +class ParserFactory: + __ParserDict = { + FileCheckConst.SHELL_SUFFIX: ShellParser(), + FileCheckConst.YAML_SUFFIX: YamlParser() + } + + def get_parser(self, file_type: str) -> Parser: + parser = self.__ParserDict.get(file_type, None) + if not parser: + raise ValueError(f'Invalid parser type: {file_type}') + return parser diff --git a/debug/accuracy_tools/msprobe/core/config_check/utils/utils.py b/debug/accuracy_tools/msprobe/core/config_check/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8c3c329cf20e2b6fb890437b3ba9950f14cc8878 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/config_check/utils/utils.py @@ -0,0 +1,107 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import hashlib + +from msprobe.core.common.framework_adapter import FmkAdp +from msprobe.core.common.log import logger + + +def merge_keys(dir_0, dir_1): + output_list = list(dir_0.keys()) + output_list.extend(list(dir_1.keys())) + return set(output_list) + + +def compare_dict(bench_dict, cmp_dict): + result = [] + for key in set(bench_dict.keys()) | set(cmp_dict.keys()): + if key in bench_dict and key in cmp_dict: + if bench_dict[key] != cmp_dict[key]: + result.append(f"{key}: {bench_dict[key]} -> {cmp_dict[key]}") + elif key in bench_dict: + result.append(f"{key}: [deleted] -> {bench_dict[key]}") + else: + result.append(f"{key}: [added] -> {cmp_dict[key]}") + return result + + +def config_checking_print(msg): + logger.info(f"[config checking log] {msg}") + + +def tensor_to_hash(tensor): + """Compute the hash value of a tensor""" + tensor_bytes = tensor.clone().detach().cpu().numpy().tobytes() + return bytes_hash(tensor_bytes) + + +def get_tensor_features(tensor): + features = { + "max": FmkAdp.tensor_max(tensor), + "min": FmkAdp.tensor_max(tensor), + "mean": FmkAdp.tensor_max(tensor), + "norm": FmkAdp.tensor_max(tensor), + } + + return features + + +def compare_dicts(dict1, dict2, path=''): + deleted = [] + added = [] + changed = [] + result = {} + + for key in dict1: + if key not in dict2: + deleted.append(f"[Deleted]: {path + key}") + result[key] = "[deleted]" + else: + if isinstance(dict1[key], dict) and isinstance(dict2[key], dict): + sub_deleted, sub_added, sub_changed, sub_result = compare_dicts( + dict1[key], dict2[key], path + key + '/') + deleted.extend(sub_deleted) + added.extend(sub_added) + changed.extend(sub_changed) + if sub_result: + result[key] = sub_result + elif dict1[key] != dict2[key]: + changed.append(f"[Changed]: {path + key} : {dict1[key]} -> {dict2[key]}") + result[key] = f"[changed]: {dict1[key]} -> {dict2[key]}" + for key in dict2: + if key not in dict1: + added.append(f"[Added]: {path + key}") + result[key] = "[added]" + return deleted, added, changed, result + + +def bytes_hash(obj: bytes): + hex_dig = hashlib.sha256(obj).hexdigest() + short_hash = int(hex_dig, 16) % (2 ** 16) + return short_hash + + +def update_dict(ori_dict, new_dict): + for key, value in new_dict.items(): + if key in ori_dict and ori_dict[key] != value: + if "values" in ori_dict.keys(): + ori_dict[key]["values"].append(new_dict[key]) + else: + ori_dict[key] = {"description": "duplicate_value", "values": [ori_dict[key], new_dict[key]]} + else: + ori_dict[key] = value diff --git a/debug/accuracy_tools/msprobe/core/data_dump/api_registry.py b/debug/accuracy_tools/msprobe/core/data_dump/api_registry.py index 1bef962232e47bc1eed399093e6812baa8f18f9c..9090c1fa206f7149d3094ac2e2066c580b6ec1f7 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/api_registry.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/api_registry.py @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect from typing import Dict, Any, Optional, Callable, Union, List, Tuple from msprobe.core.common.const import Const from msprobe.core.common.file_utils import load_yaml +from msprobe.core.common.log import logger def _get_attr(module, attr_name): @@ -32,7 +34,8 @@ def _get_attr(module, attr_name): class ApiWrapper: def __init__( self, api_types: Dict[str, Dict[str, Any]], - api_list_paths: Union[str, List[str], Tuple[str]] + api_list_paths: Union[str, List[str], Tuple[str]], + backlist: Union[List[str], Tuple[str]] = None ): self.api_types = api_types if not isinstance(api_list_paths, (list, tuple)): @@ -41,9 +44,42 @@ class ApiWrapper: raise RuntimeError("The number of api_list_paths must be equal to the number of frameworks in 'api_types', " "when api_list_paths is a list or tuple.") self.api_list_paths = api_list_paths + self.backlist = backlist if backlist else [] self.api_names = self._get_api_names() self.wrapped_api_functions = dict() + @staticmethod + def deal_with_self_kwargs(api_name, api_func, args, kwargs): + if kwargs and 'self' in kwargs: + func_params = None + try: + func_params = inspect.signature(api_func).parameters + except Exception: + if api_name in Const.API_WITH_SELF_ARG: + func_params = inspect.signature(Const.API_WITH_SELF_ARG.get(api_name)).parameters + if func_params is None: + return False, args, kwargs + + for name, param in func_params.items(): + if name == 'self' and param.kind == inspect.Parameter.KEYWORD_ONLY: + return False, args, kwargs + args_ = list(args) + names_and_values = [] + self_index = 0 + for i, item in enumerate(func_params.items()): + names_and_values.append((item[0], item[1].default)) + if item[0] == 'self': + self_index = i + break + for i in range(len(args), self_index + 1): + if names_and_values[i][0] in kwargs: + args_.append(kwargs.pop(names_and_values[i][0])) + else: + args_.append(names_and_values[i][1]) + args = tuple(args_) + + return True, args, kwargs + def wrap_api( self, api_templates, hook_build_func: Optional[Callable] ): @@ -68,6 +104,14 @@ class ApiWrapper: if callable(ori_api): def wrap_api_func(api_name, api_func, prefix, hook_build_func, api_template): def api_function(*args, **kwargs): + api_name_with_prefix = prefix + Const.SEP + str(api_name.split(Const.SEP)[-1]) + enable_wrap, args, kwargs = self.deal_with_self_kwargs(api_name_with_prefix, + api_func, args, kwargs) + if not enable_wrap: + logger.warning(f'Cannot collect precision data of {api_name_with_prefix}. ' + 'It may be fixed by passing the value of "self" ' + 'as a positional argument instead of a keyword argument. ') + return api_func(*args, **kwargs) return api_template(api_name, api_func, prefix, hook_build_func)(*args, **kwargs) api_function.__name__ = api_name return api_function @@ -84,9 +128,12 @@ class ApiWrapper: api_list = load_yaml(self.api_list_paths[index]) valid_names = dict() for api_type, api_modules in self.api_types.get(framework, {}).items(): - api_from_file = api_list.get(Const.SUPPORT_API_DICT_KEY_MAP.get(framework, {}).get(api_type), []) + key_in_file = Const.SUPPORT_API_DICT_KEY_MAP.get(framework, {}).get(api_type) + api_from_file = api_list.get(key_in_file, []) names = set() for api_name in api_from_file: + if f'{key_in_file}.{api_name}' in self.backlist: + continue target_attr = api_name target_module = api_modules[0] if Const.SEP in api_name: @@ -105,7 +152,7 @@ class ApiRegistry: Base class for api registry. """ - def __init__(self, api_types, inner_used_api, supported_api_list_path, api_templates): + def __init__(self, api_types, inner_used_api, supported_api_list_path, api_templates, backlist=None): self.ori_api_attr = dict() self.wrapped_api_attr = dict() self.inner_used_ori_attr = dict() @@ -114,6 +161,8 @@ class ApiRegistry: self.inner_used_api = inner_used_api self.supported_api_list_path = supported_api_list_path self.api_templates = api_templates + self.backlist = backlist if backlist else [] + self.all_api_registered = False @staticmethod def store_ori_attr(ori_api_group, api_list, api_ori_attr): @@ -131,7 +180,20 @@ class ApiRegistry: else: setattr(api_group, api, api_attr) + @staticmethod + def register_custom_api(module, api_name, api_prefix, hook_build_func, api_template): + def wrap_api_func(api_name, api_func, prefix, hook_build_func, api_template): + def api_function(*args, **kwargs): + return api_template(api_name, api_func, prefix, hook_build_func)(*args, **kwargs) + + api_function.__name__ = api_name + return api_function + + setattr(module, api_name, + wrap_api_func(api_name, getattr(module, api_name), api_prefix, hook_build_func, api_template)) + def register_all_api(self): + self.all_api_registered = True for framework, api_types in self.api_types.items(): for api_type, api_modules in api_types.items(): api_type_with_framework = framework + Const.SEP + api_type @@ -143,6 +205,7 @@ class ApiRegistry: self.set_api_attr(self.inner_used_api.get(api_type)[0], self.inner_used_wrapped_attr.get(api_type, {})) def restore_all_api(self): + self.all_api_registered = False for framework, api_types in self.api_types.items(): for api_type, api_modules in api_types.items(): api_type_with_framework = framework + Const.SEP + api_type @@ -154,7 +217,7 @@ class ApiRegistry: self.set_api_attr(self.inner_used_api.get(api_type)[0], self.inner_used_ori_attr.get(api_type, {})) def initialize_hook(self, hook_build_func): - api_wrapper = ApiWrapper(self.api_types, self.supported_api_list_path) + api_wrapper = ApiWrapper(self.api_types, self.supported_api_list_path, self.backlist) wrapped_api_functions = api_wrapper.wrap_api(self.api_templates, hook_build_func) for framework, api_types in self.api_types.items(): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 20e4489f89e4bd345595e6a1db1e39ab427d4908..01bebcabcfe69e1de49e6425e88696f7ac093eea 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -41,7 +41,7 @@ class DataCollector: self.backward_module_names = {} self.optimizer_status = "" self.optimizer_status_first_start = {Const.OPTIMIZER: True, Const.CLIP_GRAD: True} - atexit.register(self.write_json) + atexit.register(self.write_json_at_exit) @property def dump_data_dir(self): @@ -78,6 +78,11 @@ class DataCollector: def write_json(self): self.data_writer.write_json() + def write_json_at_exit(self): + if self.config.async_dump and self.config.task == Const.TENSOR: + self.data_processor.dump_async_data() + self.data_writer.write_json() + def update_data(self, name, data_info): msg = f"msprobe is collecting data on {name}." if self.config.task == Const.OVERFLOW_CHECK: @@ -89,6 +94,10 @@ class DataCollector: logger.debug(msg) self.data_writer.update_data(data_info) + def call_stack_collect(self, name): + stack_info = self.data_processor.analyze_api_call_stack(name) + self.data_writer.update_stack(name, stack_info) + def forward_input_data_collect(self, name, module, pid, module_input_output, is_recompute=None): if self.config.task == Const.FREE_BENCHMARK: backward_name = name.replace(Const.FORWARD, Const.BACKWARD) @@ -118,9 +127,16 @@ class DataCollector: self.set_is_recomputable(data_info, is_recompute) if self.config.level == Const.LEVEL_L2: return - self.data_writer.update_stack(self.data_processor.analyze_api_call_stack(name)) + self.call_stack_collect(name) self.handle_data(name, data_info, flush=self.data_processor.is_terminated) + def forward_data_collect_only_tensor(self, name, module, pid, module_input_output): + if not self.check_scope_and_pid(self.scope, name, pid): + return + + self.data_processor.analyze_forward(name, module, module_input_output) + + def forward_data_collect(self, name, module, pid, module_input_output, is_recompute=None): self.update_construct(name) if not self.check_scope_and_pid(self.scope, name, pid): @@ -130,9 +146,15 @@ class DataCollector: if self.config.task != Const.STRUCTURE: data_info = self.data_processor.analyze_forward(name, module, module_input_output) self.set_is_recomputable(data_info, is_recompute) - self.data_writer.update_stack(self.data_processor.analyze_api_call_stack(name)) + self.call_stack_collect(name) self.handle_data(name, data_info, flush=self.data_processor.is_terminated) + def backward_data_collect_only_tensor(self, name, module, pid, module_input_output, is_recompute=None): + if not self.check_scope_and_pid(self.scope, name, pid): + return + + self.data_processor.analyze_backward(name, module, module_input_output) + def backward_data_collect(self, name, module, pid, module_input_output, is_recompute=None): self.update_construct(name) if not self.check_scope_and_pid(self.scope, name, pid): @@ -180,7 +202,10 @@ class DataCollector: self.optimizer_status_first_start[self.optimizer_status] = False self.data_writer.update_construct({name: self.optimizer_status}) else: - self.data_writer.update_construct({name: self.module_processor.api_parent_node}) + if self.config.level == Const.LEVEL_MIX and \ + not (name.startswith(Const.MODULE) or name.startswith(Const.CELL)): + self.data_writer.update_construct({name: self.module_processor.api_parent_node}) + self.data_writer.update_construct(self.module_processor.module_node) def handle_data(self, name, data_info, flush=False): @@ -204,6 +229,7 @@ class DataCollector: def params_data_collect(self, name, param_name, pid, data): grad_name = name + Const.SEP + Const.PARAMS_GRAD + self.update_api_or_module_name(grad_name) # 校验scope和pid，以及当前name是否有过反向计算 if not self.check_scope_and_pid(self.scope, name, pid) and not self.backward_module_names.get(name): # 如果没有反向计算，则需要清除之前占位写入的grad数据 @@ -213,18 +239,19 @@ class DataCollector: data_info = self.data_processor.analyze_params(grad_name, param_name, data) self.handle_data(grad_name, data_info, flush=self.data_processor.is_terminated) - def fill_stack_tensor_data(self): - self.data_writer.fill_stack_tensor_data() def debug_data_collect_forward(self, variable, name_with_count): data_info = self.data_processor.analyze_debug_forward(variable, name_with_count) - self.data_writer.update_debug({name_with_count: data_info}) + name_with_count_category = name_with_count + Const.SEP + Const.DEBUG + self.data_writer.update_debug({name_with_count_category: data_info}) def debug_data_collect_backward(self, variable, grad_name_with_count): # prepare all None nested data structure all_none_data_info = self.data_processor.analyze_element_to_all_none(variable) - self.data_writer.update_debug({grad_name_with_count: all_none_data_info}) + grad_name_with_count_category = grad_name_with_count + Const.SEP + Const.DEBUG + self.data_writer.update_debug({grad_name_with_count_category: all_none_data_info}) # register tensor backward hook - self.data_processor.analyze_debug_backward(variable, grad_name_with_count, self.data_writer.cache_debug['data']) + self.data_processor.analyze_debug_backward(variable, grad_name_with_count_category, + self.data_writer.cache_debug['data']) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 775a80b2418ef356867228b4ca09fad8c86cce25..60257b14b2ec2a5958d771e36e10c349f79aaaac 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,17 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import inspect import os from dataclasses import dataclass, is_dataclass -from typing import Tuple, Dict, Optional, Any from functools import partial -import copy -from typing import Union +from typing import Tuple, Dict, Optional, Any, Union import numpy as np from msprobe.core.common.const import Const +from msprobe.core.common.file_utils import save_npy from msprobe.core.common.log import logger from msprobe.core.common.utils import convert_tuple, CompareException @@ -79,21 +79,17 @@ class ModuleBackwardOutputs: class TensorStatInfo: - def __init__(self, max_val=None, min_val=None, mean_val=None, norm_val=None, stack_tensor_stat=None): + def __init__(self, max_val=None, min_val=None, mean_val=None, norm_val=None): self.max = max_val self.min = min_val self.mean = mean_val self.norm = norm_val - self.stack_tensor_stat = stack_tensor_stat class BaseDataProcessor: _recursive_key_stack = [] - special_type = ( - np.integer, np.floating, np.bool_, np.complexfloating, np.str_, np.byte, np.unicode_, np.ndarray, - bool, int, float, str, slice, - type(Ellipsis) - ) + builtin_type = (bool, int, float, str, slice, type(Ellipsis)) + np_type = (np.integer, np.floating, np.bool_, np.complexfloating, np.str_, np.byte, np.unicode_, np.ndarray) def __init__(self, config, data_writer): self.data_writer = data_writer @@ -120,7 +116,10 @@ class BaseDataProcessor: @staticmethod def analyze_api_call_stack(name): try: - api_stack = inspect.stack()[5:] + if name.startswith("Primitive"): + api_stack = inspect.stack()[4:] + else: + api_stack = inspect.stack()[5:] except Exception as e: logger.warning(f"The call stack of <{name}> failed to retrieve, {e}.") api_stack = None @@ -129,12 +128,14 @@ class BaseDataProcessor: for (_, path, line, func, code, _) in api_stack: if not code: continue + if any(filter_path in path for filter_path in Const.STACK_FILTER_KEYWORDS) and \ + Const.CALL_STACK_FLAG not in path: + continue stack_line = f"File {path}, line {str(line)}, in {func}, \n {code[0].strip()}" stack_str.append(stack_line) else: stack_str.append(Const.WITHOUT_CALL_STACK) - stack_info_struct = {name: stack_str} - return stack_info_struct + return tuple(stack_str) @staticmethod def transfer_type(data): @@ -178,20 +179,8 @@ class BaseDataProcessor: "invalid data_structure type or invalid index") @staticmethod - def _convert_numpy_to_builtin(arg): - type_mapping = { - np.integer: int, - np.floating: float, - np.bool_: bool, - np.complexfloating: complex, - np.str_: str, - np.byte: bytes, - np.unicode_: str - } - for numpy_type, builtin_type in type_mapping.items(): - if isinstance(arg, numpy_type): - return builtin_type(arg), type(arg).__name__ - return arg, '' + def is_distributed_op(module): + return getattr(module, "op_is_distributed", False) @staticmethod def _analyze_builtin(arg): @@ -217,7 +206,11 @@ class BaseDataProcessor: return single_arg @staticmethod - def _analyze_numpy(ndarray, numpy_type): + def _analyze_numpy(arg): + return {"type": type(arg).__name__, "value": arg.item()} + + @staticmethod + def _analyze_ndarray(ndarray, _): ndarray_json = {} ndarray_json.update({'type': 'numpy.ndarray'}) ndarray_json.update({'dtype': str(ndarray.dtype)}) @@ -248,12 +241,12 @@ class BaseDataProcessor: @classmethod def get_special_types(cls): - return cls.special_type + return cls.builtin_type + cls.np_type @classmethod def recursive_apply_transform(cls, args, transform, depth=0) -> Union[dict, list, None]: - if depth > Const.MAX_DEPTH: - logger.error(f"The maximum depth of recursive transform, {Const.MAX_DEPTH} is reached.") + if depth > Const.DUMP_MAX_DEPTH: + logger.error(f"The maximum depth of recursive transform, {Const.DUMP_MAX_DEPTH} is reached.") raise CompareException(CompareException.RECURSION_LIMIT_ERROR) if isinstance(args, cls.get_special_types()): arg_transform = transform(args, cls._recursive_key_stack) @@ -303,6 +296,7 @@ class BaseDataProcessor: def real_hook_fn(grad): return wrap_hook_fn(grad) + element.register_hook(real_hook_fn) def if_return_forward_new_output(self): @@ -350,6 +344,8 @@ class BaseDataProcessor: return api_info_struct def analyze_forward_output(self, name, module, module_input_output: ModuleForwardInputsOutputs): + if self.is_distributed_op(module): + module_input_output.update_output_with_args_and_kwargs() api_info_struct = {} # check whether data_mode contains forward or input if self.is_dump_for_data_mode(Const.FORWARD, Const.OUTPUT): @@ -427,6 +423,7 @@ class BaseDataProcessor: api_info_struct = {} self.save_name = name + Const.SEP + param_name data_info = self.analyze_element(grad) + self.save_name = None grad_info_dict = {param_name: [data_info]} api_info_struct[name] = grad_info_dict return api_info_struct @@ -435,10 +432,10 @@ class BaseDataProcessor: file_format = Const.PT_SUFFIX if self.config.framework == Const.PT_FRAMEWORK else Const.NUMPY_SUFFIX if self.save_name is not None: dump_data_name = (self.save_name + file_format) - self.save_name = None else: - dump_data_name = (self.current_api_or_module_name + Const.SEP + self.api_data_category + Const.SEP + - suffix + file_format) + suffix_with_seq = (Const.SEP + suffix) if suffix else "" + dump_data_name = (self.current_api_or_module_name + Const.SEP + self.api_data_category + suffix_with_seq + + file_format) file_path = os.path.join(self.data_writer.dump_tensor_data_dir, dump_data_name) return dump_data_name, file_path @@ -447,23 +444,32 @@ class BaseDataProcessor: def analyze_debug_forward(self, variable, name_with_count): self.current_api_or_module_name = name_with_count - self.api_data_category = Const.TENSOR - # these two attributes are used to construct tensor file name {name_with_count}.tensor.{indexes}.npy/pt + self.api_data_category = Const.DEBUG + # these two attributes are used to construct tensor file name {name_with_count}.debug.{indexes}.npy/pt data_info = self.analyze_element(variable) return data_info - def analyze_debug_backward(self, variable, grad_name_with_count, nested_data_structure): + def analyze_debug_backward(self, variable, grad_name_with_count_category, nested_data_structure): def hook_fn(grad, indexes): suffix = Const.SEP.join([str(index) for index in indexes]) - self.save_name = grad_name_with_count + Const.SEP + Const.TENSOR + Const.SEP + suffix + suffix_with_sep = (Const.SEP + suffix) if suffix else "" + self.save_name = grad_name_with_count_category + suffix_with_sep grad_data_info = self.analyze_element(grad) self.save_name = None - full_index = [grad_name_with_count] + indexes + full_index = [grad_name_with_count_category] + indexes try: self.set_value_into_nested_structure(nested_data_structure, full_index, grad_data_info) except (ValueError, IndexError) as e: - logger.warning(f"error occured while recording statistics of {grad_name_with_count} variable, " - f"skip current recording, detailed infomation: {e}") + logger.warning(f"error occurred while recording statistics of {grad_name_with_count_category} variable," + f"skip current recording, detailed information: {e}") return grad + wrap_register_hook_single_element = partial(self.register_hook_single_element, hook_fn=hook_fn) - self.recursive_apply_transform(variable, wrap_register_hook_single_element) \ No newline at end of file + self.recursive_apply_transform(variable, wrap_register_hook_single_element) + + def _analyze_and_save_ndarray(self, ndarray, suffix): + dump_data_name, file_path = self.get_save_file_path(suffix) + save_npy(ndarray, file_path) + ndarray_json = BaseDataProcessor._analyze_ndarray(ndarray, suffix) + ndarray_json.update({"data_name": dump_data_name}) + return ndarray_json diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index c6ab0293cf3edafab06a5bf03e1a429d86e92720..5a1e7569d8fc3423da7664dbd7582858e75ad062 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -17,13 +17,14 @@ import zlib import mindspore as ms from mindspore import mint, ops, hal +from mindspore.mint import distributed from mindspore._c_expression.typing import Number import numpy as np from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import (BaseDataProcessor, TensorStatInfo, ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs) -from msprobe.core.common.file_utils import path_len_exceeds_limit, save_npy +from msprobe.core.common.file_utils import path_len_exceeds_limit from msprobe.mindspore.common.utils import convert_bf16_to_fp32, save_tensor_as_npy from msprobe.mindspore.common.log import logger from msprobe.mindspore.dump.hook_cell.api_register import get_api_register @@ -36,7 +37,7 @@ except ImportError: class MindsporeDataProcessor(BaseDataProcessor): - mindspore_special_type = tuple([ms.Tensor, Number]) + mindspore_special_type = tuple([ms.Tensor, Number, distributed.P2POp]) def __init__(self, config, data_writer): super().__init__(config, data_writer) @@ -65,7 +66,7 @@ class MindsporeDataProcessor(BaseDataProcessor): tensor_stat.max = np.max(data_np).item() tensor_stat.min = np.min(data_np).item() elif not data.shape: - tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data.item() + tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data elif data.dtype == ms.complex64 or data.dtype == ms.complex128: data_abs = np.abs(data.asnumpy()) tensor_stat.max = np.max(data_abs).item() @@ -76,38 +77,52 @@ class MindsporeDataProcessor(BaseDataProcessor): if not ops.is_floating_point(data) or data.dtype == ms.float64: data = data.to(ms.float32) get_norm_value = mint.norm if hasattr(mint, "norm") else ops.norm - tensor_stat.max = mint.max(data).item() - tensor_stat.min = mint.min(data).item() - tensor_stat.mean = mint.mean(data).item() - tensor_stat.norm = get_norm_value(data).item() + tensor_stat.max = mint.max(data) + tensor_stat.min = mint.min(data) + tensor_stat.mean = mint.mean(data) + tensor_stat.norm = get_norm_value(data) return tensor_stat @staticmethod def get_stat_info_async(data): tensor_stat = TensorStatInfo() - if data.dtype == ms.complex64 or data.dtype == ms.complex128: + if data.dtype == ms.bool_: + tensor_stat.max = mint.any(data) + tensor_stat.min = mint.all(data) + elif not data.shape: + tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data + elif data.dtype == ms.complex64 or data.dtype == ms.complex128: logger.warning("Async dump do not support complex data!") return tensor_stat - elif data.dtype == ms.bool_: - tensor_stat.stack_tensor_stat = (["Max", "Min"], ops.stack([data.any(), data.all()])) - elif not data.shape: - tensor_stat.stack_tensor_stat = (["Max", "Min", "Mean", "Norm"], ops.stack([data, data, data, data])) else: if not ops.is_floating_point(data) or data.dtype == ms.float64: data = data.to(ms.float32) get_norm_value = mint.norm if hasattr(mint, "norm") else ops.norm - tensor_stat.stack_tensor_stat = (["Max", "Min", "Mean", "Norm"], ops.stack( - [mint.max(data), mint.min(data), mint.mean(data), get_norm_value(data)])) + tensor_stat.max = mint.max(data) + tensor_stat.min = mint.min(data) + tensor_stat.mean = mint.mean(data) + tensor_stat.norm = get_norm_value(data) return tensor_stat @staticmethod def is_hookable_element(element): return hasattr(element, "register_hook") and callable(element.register_hook) + @staticmethod + def process_group_hash(arg): + group_ranks = distributed.get_process_group_ranks(arg) + group_ranks_hash = zlib.crc32(str(group_ranks).encode('utf-8')) + return f"{group_ranks_hash:08x}" + @classmethod def get_special_types(cls): return super().get_special_types() + cls.mindspore_special_type + def dump_async_data(self): + for file_path, tensor in self._async_dump_cache.items(): + save_tensor_as_npy(tensor, file_path) + self._async_dump_cache.clear() + def get_stat_info(self, data): self.api_register.restore_inner_used_api() tensor_stat = TensorStatInfo() @@ -125,19 +140,34 @@ class MindsporeDataProcessor(BaseDataProcessor): if suffix_stack and suffix_stack[-1] in self.mindspore_object_key: return self.mindspore_object_key[suffix_stack[-1]](element) - converted_numpy, numpy_type = self._convert_numpy_to_builtin(element) - if converted_numpy is not element: - return {"type": numpy_type, "value": converted_numpy} - if isinstance(element, Number): - return self.analyze_dtype_in_kwargs(element) - if isinstance(element, ms.Tensor): - return self._analyze_tensor(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) - if isinstance(element, np.ndarray): - return self._analyze_numpy(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) - if isinstance(element, (bool, int, float, str, slice, type(Ellipsis))): - return self._analyze_builtin(element) + suffix_str = Const.SEP.join(str(s) for s in suffix_stack) + type_analyzer = [ + (MindsporeDataProcessor.builtin_type, self._analyze_builtin), + (ms.Tensor, lambda e: self._analyze_tensor(e, suffix_str)), + (Number, self.analyze_dtype_in_kwargs), + (MindsporeDataProcessor.np_type[:-1], self._analyze_numpy), + (np.ndarray, lambda e: self._analyze_ndarray(e, suffix_str)), + (distributed.P2POp, lambda e: self._analyze_p2pop(e, suffix_str)) + ] + for type_key, analyze_fn in type_analyzer: + if isinstance(element, type_key): + return analyze_fn(element) return {} + def _analyze_p2pop(self, arg, suffix): + p2pop_info = {"class_type": "mindspore.mint.distributed.P2POp"} + try: + tensor_info = self._analyze_tensor(arg.tensor, suffix) + p2pop_info.update({"tensor": tensor_info}) + p2pop_info.update({"op": arg.op}) + p2pop_info.update({"peer": arg.peer}) + p2pop_info.update({"tag": arg.tag}) + group_id = self.process_group_hash(arg.group) if arg.group else None + p2pop_info.update({"group_id": group_id}) + except Exception as e: + logger.warning(f"Failed to parse the P2POp content with error info: {e}.") + return p2pop_info + def _analyze_tensor(self, tensor, suffix): tensor_stat = self.get_stat_info(tensor) tensor_json = { @@ -146,32 +176,26 @@ class MindsporeDataProcessor(BaseDataProcessor): 'shape': tensor.shape } - if tensor_stat.stack_tensor_stat is None: - tensor_json.update({'Max': self.transfer_type(tensor_stat.max)}) - tensor_json.update({'Min': self.transfer_type(tensor_stat.min)}) - tensor_json.update({'Mean': self.transfer_type(tensor_stat.mean)}) - tensor_json.update({'Norm': self.transfer_type(tensor_stat.norm)}) - else: - tensor_json.update({'tensor_stat': tensor_stat.stack_tensor_stat}) + # 将统计值存入全局 buffer，并返回占位索引 + stat_values = [ + tensor_stat.max, + tensor_stat.min, + tensor_stat.mean, + tensor_stat.norm + ] + + placeholder_index = self.data_writer.append_stat_to_buffer(stat_values) + + tensor_json.update({Const.TENSOR_STAT_INDEX: placeholder_index}) + if self.config.summary_mode == Const.MD5 and not self.config.async_dump: tensor_md5 = self.get_md5_for_tensor(tensor) tensor_json.update({Const.MD5: tensor_md5}) return tensor_json - -class StatisticsDataProcessor(MindsporeDataProcessor): - pass - - -class TensorDataProcessor(MindsporeDataProcessor): - def dump_async_data(self): - for file_path, tensor in self._async_dump_cache.items(): - save_tensor_as_npy(tensor, file_path) - self._async_dump_cache.clear() - - def _analyze_tensor(self, tensor, suffix): + def _analyze_and_save_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) - single_arg = super()._analyze_tensor(tensor, suffix) + single_arg = MindsporeDataProcessor._analyze_tensor(self, tensor, suffix) single_arg.update({"data_name": dump_data_name}) if self.config.async_dump: self._async_dump_cache[file_path] = tensor.copy() @@ -179,12 +203,27 @@ class TensorDataProcessor(MindsporeDataProcessor): save_tensor_as_npy(tensor, file_path) return single_arg - def _analyze_numpy(self, ndarray, suffix): - dump_data_name, file_path = self.get_save_file_path(suffix) - save_npy(ndarray, file_path) - ndarray_json = super()._analyze_numpy(ndarray, suffix) - ndarray_json.update({"data_name": dump_data_name}) - return ndarray_json + +class StatisticsDataProcessor(MindsporeDataProcessor): + def _analyze_tensor(self, tensor, suffix): + if any(item in self.current_api_or_module_name for item in self.config.tensor_list): + return self._analyze_and_save_tensor(tensor, suffix) + else: + return super()._analyze_tensor(tensor, suffix) + + def _analyze_ndarray(self, ndarray, suffix): + if any(item in self.current_api_or_module_name for item in self.config.tensor_list): + return self._analyze_and_save_ndarray(ndarray, suffix) + else: + return super()._analyze_ndarray(ndarray, suffix) + + +class TensorDataProcessor(MindsporeDataProcessor): + def _analyze_tensor(self, tensor, suffix): + return self._analyze_and_save_tensor(tensor, suffix) + + def _analyze_ndarray(self, ndarray, suffix): + return self._analyze_and_save_ndarray(ndarray, suffix) class OverflowCheckDataProcessor(MindsporeDataProcessor): @@ -231,7 +270,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): api_info_struct = super().analyze_backward(name, module, module_input_output) self.maybe_save_overflow_data() return api_info_struct if self.has_overflow else None - + def analyze_params(self, name, param_name, grad): self.has_overflow = False api_info_struct = super().analyze_params(name, param_name, grad) @@ -249,11 +288,26 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): self.cached_tensors_and_file_paths = {} def _analyze_maybe_overflow_tensor(self, tensor_json): - if tensor_json['Max'] is None: + tensor_stat_index = tensor_json.get(Const.TENSOR_STAT_INDEX) + if tensor_stat_index is None: + logger.warning("tensor_stat_index does not exist in tensor_json.") return - if np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']): + max_tensor = self.data_writer.get_buffer_values_max(tensor_stat_index) + min_tensor = self.data_writer.get_buffer_values_min(tensor_stat_index) + if max_tensor is None or min_tensor is None: + return + + def check_inf_nan(value): + # Use .item() if it's a tensor-like structure + if hasattr(value, "item"): + value = value.item() + return np.isinf(value) or np.isnan(value) + + if check_inf_nan(max_tensor): self.has_overflow = True - if np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']): + return + + if check_inf_nan(min_tensor): self.has_overflow = True def _analyze_tensor(self, tensor, suffix): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index acab9323071c7ce552189509b5b706a6df48c8dc..2cd93b3caeea7acbb13ec18a56351284de7602da 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib import zlib from dataclasses import asdict from typing import List @@ -27,7 +26,8 @@ from msprobe.core.common.const import Const from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common.file_utils import path_len_exceeds_limit from msprobe.core.common.log import logger -from msprobe.core.common.utils import convert_tuple, recursion_depth_decorator +from msprobe.core.common.utils import convert_tuple +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, ModuleBackwardInputsOutputs, \ ModuleForwardInputsOutputs, TensorStatInfo from msprobe.pytorch.common.utils import Const as PtConst, save_pt, is_hifloat8_tensor, is_float8_tensor @@ -101,19 +101,17 @@ class PytorchDataProcessor(BaseDataProcessor): logger.warning("Async dump do not support complex data!") return tensor_stat elif data.dtype == torch.bool: - tensor_stat.stack_tensor_stat = (["Max", "Min"], torch.stack( - [torch.any(data), torch.all(data)])) + tensor_stat.max = torch.any(data) + tensor_stat.min = torch.all(data) elif not data.shape: - tensor_stat.stack_tensor_stat = (["Max", "Min", "Mean", "Norm"], torch.stack([data, data, data, data])) + tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data else: - if not data.is_floating_point() or data.dtype == torch.float64: + if data.dtype == torch.float64 or not data.is_floating_point(): data = data.float() - tensor_stat.stack_tensor_stat = (["Max", "Min", "Mean", "Norm"], torch.stack([ - torch.max(data), - torch.min(data), - torch.mean(data), - torch.norm(data) - ])) + tensor_stat.max = torch.max(data) + tensor_stat.min = torch.min(data) + tensor_stat.mean = torch.mean(data) + tensor_stat.norm = torch.norm(data) return tensor_stat @staticmethod @@ -126,17 +124,17 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_stat.min = np.min(data_abs).item() tensor_stat.mean = np.mean(data_abs).item() elif data.dtype == torch.bool: - tensor_stat.max = torch.any(data).item() - tensor_stat.min = torch.all(data).item() + tensor_stat.max = torch.any(data) + tensor_stat.min = torch.all(data) elif not data.shape: - tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data.item() + tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data else: - if not data.is_floating_point() or data.dtype == torch.float64: + if data.dtype == torch.float64 or not data.is_floating_point(): data = data.float() - tensor_stat.max = torch.max(data).item() - tensor_stat.min = torch.min(data).item() - tensor_stat.mean = torch.mean(data).item() - tensor_stat.norm = torch.norm(data).item() + tensor_stat.max = torch.max(data) + tensor_stat.min = torch.min(data) + tensor_stat.mean = torch.mean(data) + tensor_stat.norm = torch.norm(data) return tensor_stat @staticmethod @@ -173,12 +171,8 @@ class PytorchDataProcessor(BaseDataProcessor): @staticmethod def process_group_hash(arg): group_ranks = dist.get_process_group_ranks(arg) - group_ranks_hash = hashlib.md5(str(group_ranks).encode('utf-8')).hexdigest() - return group_ranks_hash - - @staticmethod - def is_distributed_op(module): - return getattr(module, "op_is_distributed", False) + group_ranks_hash = zlib.crc32(str(group_ranks).encode('utf-8')) + return f"{group_ranks_hash:08x}" @staticmethod def is_hookable_element(element): @@ -232,34 +226,31 @@ class PytorchDataProcessor(BaseDataProcessor): def get_special_types(cls): return super().get_special_types() + cls.pytorch_special_type + def dump_async_data(self): + for file_path, tensor in self._async_dump_cache.items(): + save_pt(tensor.contiguous(), file_path) + self._async_dump_cache.clear() + def analyze_single_element(self, element, suffix_stack): if suffix_stack and suffix_stack[-1] in self.torch_object_key: return self.torch_object_key[suffix_stack[-1]](element) - if isinstance(element, torch.Size): - return self._analyze_torch_size(element) - if isinstance(element, torch.memory_format): - return self._analyze_memory_format(element) - if isinstance(element, dist.ProcessGroup): - return self._analyze_process_group(element) - if isinstance(element, dist.P2POp): - return self._analyze_p2pop(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) - if isinstance(element, dist.ReduceOp): - return self._analyze_reduce_op(element) - converted_numpy, numpy_type = self._convert_numpy_to_builtin(element) - if converted_numpy is not element: - return {"type": numpy_type, "value": converted_numpy} - if isinstance(element, torch.Tensor): - return self._analyze_tensor(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) - if isinstance(element, np.ndarray): - return self._analyze_numpy(element, Const.SEP.join([str(suffix) for suffix in suffix_stack])) - if isinstance(element, (bool, int, float, str, slice, type(Ellipsis))): - return self._analyze_builtin(element) - return {} - def analyze_forward_output(self, name, module, module_input_output: ModuleForwardInputsOutputs): - if self.is_distributed_op(module): - module_input_output.update_output_with_args_and_kwargs() - return super().analyze_forward_output(name, module, module_input_output) + suffix_str = Const.SEP.join(str(s) for s in suffix_stack) + type_analyzer = [ + (PytorchDataProcessor.builtin_type, self._analyze_builtin), + (torch.Size, self._analyze_torch_size), + (torch.Tensor, lambda e: self._analyze_tensor(e, suffix_str)), + (torch.memory_format, self._analyze_memory_format), + (dist.ProcessGroup, self._analyze_process_group), + (dist.P2POp, lambda e: self._analyze_p2pop(e, suffix_str)), + (dist.ReduceOp, self._analyze_reduce_op), + (PytorchDataProcessor.np_type[:-1], self._analyze_numpy), + (np.ndarray, lambda e: self._analyze_ndarray(e, suffix_str)), + ] + for type_key, analyze_fn in type_analyzer: + if isinstance(element, type_key): + return analyze_fn(element) + return {} def _analyze_p2pop(self, arg, suffix): p2pop_info = {"class_type": "torch.distributed.P2POp"} @@ -283,42 +274,26 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_json.update({'type': 'torch.Tensor'}) tensor_json.update({'dtype': dtype}) tensor_json.update({"shape": tensor.shape}) - if tensor_stat.stack_tensor_stat is None: - tensor_json.update({"Max": tensor_stat.max}) - tensor_json.update({"Min": tensor_stat.min}) - tensor_json.update({"Mean": tensor_stat.mean}) - tensor_json.update({"Norm": tensor_stat.norm}) - tensor_json.update({"requires_grad": tensor.requires_grad}) - if tensor_stat.max is not None: - if np.isinf(tensor_stat.max) or np.isnan(tensor_stat.max): - tensor_json['Max_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(tensor, "max") - if tensor_stat.min is not None: - if np.isinf(tensor_stat.min) or np.isnan(tensor_stat.min): - tensor_json['Min_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(tensor, "min") - else: - tensor_json.update({"requires_grad": tensor.requires_grad}) - tensor_json.update({"tensor_stat": tensor_stat.stack_tensor_stat}) + stat_values = [ + tensor_stat.max, + tensor_stat.min, + tensor_stat.mean, + tensor_stat.norm + ] + placeholder_index = self.data_writer.append_stat_to_buffer(stat_values) + + tensor_json.update({Const.TENSOR_STAT_INDEX: placeholder_index}) + tensor_json.update({"requires_grad": tensor.requires_grad}) if self.config.summary_mode == Const.MD5 and not self.config.async_dump: tensor_md5 = self.get_md5_for_tensor(tensor) tensor_json.update({Const.MD5: tensor_md5}) return tensor_json - -class StatisticsDataProcessor(PytorchDataProcessor): - pass - - -class TensorDataProcessor(PytorchDataProcessor): - def dump_async_data(self): - for file_path, tensor in self._async_dump_cache.items(): - save_pt(tensor.contiguous(), file_path) - self._async_dump_cache.clear() - - def _analyze_tensor(self, tensor, suffix): + def _analyze_and_save_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) - single_arg = super()._analyze_tensor(tensor, suffix) + single_arg = PytorchDataProcessor._analyze_tensor(self, tensor, suffix) single_arg.update({"data_name": dump_data_name}) tensor, _ = self._cast_to_float_if_fp8(tensor) if self.config.async_dump: @@ -328,14 +303,36 @@ class TensorDataProcessor(PytorchDataProcessor): save_pt(saved_tensor, file_path) return single_arg - def _analyze_numpy(self, ndarray, suffix): + def _analyze_and_save_ndarray(self, ndarray, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) save_pt(torch.tensor(ndarray), file_path) - ndarray_json = super()._analyze_numpy(ndarray, suffix) + ndarray_json = PytorchDataProcessor._analyze_ndarray(ndarray, suffix) ndarray_json.update({"data_name": dump_data_name}) return ndarray_json +class StatisticsDataProcessor(PytorchDataProcessor): + def _analyze_tensor(self, tensor, suffix): + if any(item in self.current_api_or_module_name for item in self.config.tensor_list): + return self._analyze_and_save_tensor(tensor, suffix) + else: + return super()._analyze_tensor(tensor, suffix) + + def _analyze_ndarray(self, ndarray, suffix): + if any(item in self.current_api_or_module_name for item in self.config.tensor_list): + return self._analyze_and_save_ndarray(ndarray, suffix) + else: + return super()._analyze_ndarray(ndarray, suffix) + + +class TensorDataProcessor(PytorchDataProcessor): + def _analyze_tensor(self, tensor, suffix): + return self._analyze_and_save_tensor(tensor, suffix) + + def _analyze_ndarray(self, ndarray, suffix): + return self._analyze_and_save_ndarray(ndarray, suffix) + + class OverflowCheckDataProcessor(PytorchDataProcessor): __slots__ = ["cached_tensors_and_file_paths"] @@ -426,10 +423,22 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): raise RuntimeError(f"overflow check failed") from e def _analyze_maybe_overflow_tensor(self, tensor_json): - if tensor_json['Max'] is None or tensor_json['Min'] is None: + tensor_stat_index = tensor_json.get(Const.TENSOR_STAT_INDEX) + if tensor_stat_index is None: + logger.warning("tensor_stat_index does not exist in tensor_json.") + return + max_tensor = self.data_writer.get_buffer_values_max(tensor_stat_index) + min_tensor = self.data_writer.get_buffer_values_min(tensor_stat_index) + + if max_tensor is None or min_tensor is None: + return + + if torch.isinf(max_tensor) or torch.isnan(max_tensor): + self.has_overflow = True return - self.has_overflow = np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']) or \ - np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']) + + if torch.isinf(min_tensor) or torch.isnan(min_tensor): + self.has_overflow = True def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) @@ -573,7 +582,10 @@ class KernelDumpDataProcessor(PytorchDataProcessor): self.stop_kernel_dump() logger.info(f"The kernel data of {name} is dumped successfully.") - @recursion_depth_decorator("KernelDump: KernelDumpDataProcessor.clone_and_detach_tensor") + @recursion_depth_decorator( + "KernelDump: KernelDumpDataProcessor.clone_and_detach_tensor", + max_depth=Const.DUMP_MAX_DEPTH + ) def clone_and_detach_tensor(self, input_params): if isinstance(input_params, torch.Tensor): if is_float8_tensor(input_params): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py index b1e26d16f9741765c1c9600a64efb112aa0f42d7..0f80c6f85266a896888a0cde17d89abb11600d8d 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,12 +16,14 @@ import csv import os import copy -import numpy as np +import threading from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.file_utils import change_mode, FileOpen, save_json, load_json from msprobe.core.common.log import logger -from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common.decorator import recursion_depth_decorator + +lock = threading.Lock() class DataWriter: @@ -34,10 +36,12 @@ class DataWriter: self.dump_tensor_data_dir = None self.debug_file_path = None self.flush_size = 1000 + self.larger_flush_size = 20000 self.cache_data = {} self.cache_stack = {} self.cache_construct = {} self.cache_debug = {} + self.stat_stack_list = [] @staticmethod def write_data_to_csv(result: list, result_header: tuple, file_path: str): @@ -54,10 +58,51 @@ class DataWriter: if is_new_file: change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + @recursion_depth_decorator("JsonWriter: DataWriter._replace_stat_placeholders") + def _replace_stat_placeholders(self, data, stat_result): + if isinstance(data, dict): + keys = list(data.keys()) # 获取当前所有键 + for key in keys: # 递归所有变量 + value = data[key] + if key == Const.TENSOR_STAT_INDEX and isinstance(value, int): + if value >= 0: + idx = value + else: + return + stat_values = stat_result[idx] if idx < len(stat_result) else [None] * 4 + + new_entries = { + Const.TYPE: data["type"], + Const.DTYPE: data["dtype"], + Const.SHAPE: data["shape"], + Const.MAX: stat_values[0], + Const.MIN: stat_values[1], + Const.MEAN: stat_values[2], + Const.NORM: stat_values[3], + } + del data[key] + + # 重构字典顺序 + updated_dict = {} + # 通过插入排序后字段保证字段写入json的有序 + updated_dict.update(new_entries) + # 遍历原字典其他字段（排除已删除的tensor_stat_index） + for k in data: + if k not in new_entries: + updated_dict[k] = data[k] + data.clear() + data.update(updated_dict) + else: + self._replace_stat_placeholders(value, stat_result) + elif isinstance(data, (list, tuple)): + for item in data: + self._replace_stat_placeholders(item, stat_result) + def reset_cache(self): self.cache_data = {} self.cache_stack = {} self.cache_construct = {} + self.cache_debug = {} def initialize_json_file(self, **kwargs): if self.debug_file_path and not self.cache_debug: @@ -86,39 +131,59 @@ class DataWriter: def flush_data_periodically(self): dump_data = self.cache_data.get(Const.DATA) - if dump_data and isinstance(dump_data, dict) and len(dump_data) % self.flush_size == 0: - self.write_json() - def update_data(self, new_data): - if not isinstance(new_data, dict) or len(new_data.keys()) != 1: - logger.warning(f"The data info({new_data}) should be a dict with only one outer key.") - return - dump_data = self.cache_data.get(Const.DATA) - if not isinstance(dump_data, dict): - logger.warning(f"The dump data({dump_data}) should be a dict.") + if not dump_data or not isinstance(dump_data, dict): return - key = next(iter(new_data.keys())) - if key in dump_data: - dump_data.get(key).update(new_data.get(key)) - else: - dump_data.update(new_data) + length = len(dump_data) - def update_stack(self, new_data): - self.cache_stack.update(new_data) + threshold = self.flush_size if length < self.larger_flush_size else self.larger_flush_size + + if length % threshold == 0: + self.write_json() + + def update_data(self, new_data): + with lock: + if not isinstance(new_data, dict) or len(new_data.keys()) != 1: + logger.warning(f"The data info({new_data}) should be a dict with only one outer key.") + return + dump_data = self.cache_data.get(Const.DATA) + if not isinstance(dump_data, dict): + logger.warning(f"The dump data({dump_data}) should be a dict.") + return + + key = next(iter(new_data.keys())) + if key in dump_data: + dump_data.get(key).update(new_data.get(key)) + else: + dump_data.update(new_data) + + def update_stack(self, name, stack_data): + with lock: + api_list = self.cache_stack.get(stack_data) + if api_list is None: + self.cache_stack.update({stack_data: [name]}) + else: + api_list.append(name) def update_construct(self, new_data): - self.cache_construct.update(new_data) + with lock: + self.cache_construct.update(new_data) def update_debug(self, new_data): - self.cache_debug['data'].update(new_data) + with lock: + self.cache_debug['data'].update(new_data) def write_data_json(self, file_path): logger.info(f"dump.json is at {os.path.dirname(os.path.dirname(file_path))}. ") save_json(file_path, self.cache_data, indent=1) def write_stack_info_json(self, file_path): - save_json(file_path, self.cache_stack, indent=1) + num, new_cache_stack = 0, {} + for key, value in self.cache_stack.items(): + new_cache_stack[num] = [value, key] + num += 1 + save_json(file_path, new_cache_stack, indent=1) def write_construct_info_json(self, file_path): save_json(file_path, self.cache_construct, indent=1) @@ -126,38 +191,62 @@ class DataWriter: def write_debug_info_json(self, file_path): save_json(file_path, self.cache_debug, indent=1) + def append_stat_to_buffer(self, stat_vector): + """ + 直接使用 Python list 存储 stat_vector, + 将 stat_vector 存入 self.stat_stack_list 的方式 + """ + self.stat_stack_list.append(stat_vector) + return len(self.stat_stack_list) - 1 + + def get_buffer_values_max(self, index): + if 0 <= index < len(self.stat_stack_list) and len(self.stat_stack_list[index]) >= 1: + return self.stat_stack_list[index][0] + else: + logger.warning(f"stat_stack_list[{index}] The internal data is incomplete," + f" and the maximum value cannot be obtained.") + return None + + def get_buffer_values_min(self, index): + if 0 <= index < len(self.stat_stack_list) and len(self.stat_stack_list[index]) >= 1: + return self.stat_stack_list[index][1] + else: + logger.warning(f"stat_stack_list[{index}] Internal data is incomplete" + f" and minimum values cannot be obtained.") + return None + + def flush_stat_stack(self): + """ + 在 flush 阶段，将所有存储的统计值从设备搬到 CPU， + 这里返回一个列表，每个元素是 [Max, Min, Mean, Norm] 的数值列表 + """ + if not self.stat_stack_list: + return [] + result = [ + [ + x.item() if hasattr(x, "item") else x + for x in stat_values + ] + for stat_values in self.stat_stack_list + ] + self.stat_stack_list = [] + return result + def write_json(self): - if self.cache_data: - self.write_data_json(self.dump_file_path) - if self.cache_stack: - self.write_stack_info_json(self.stack_file_path) - if self.cache_construct: - self.write_construct_info_json(self.construct_file_path) - if self.cache_debug: - self.write_debug_info_json(self.debug_file_path) - - def fill_stack_tensor_data(self): - self.process_stat_data_recursive(self.cache_data) - - def process_stat_data_recursive(self, data, depth=0): - if depth > Const.MAX_DEPTH: - logger.error(f"The maximum depth of recursive process stat data, {Const.MAX_DEPTH} is reached.") - raise MsprobeException(MsprobeException.RECURSION_LIMIT_ERROR) - if isinstance(data, dict): - if "tensor_stat" in data.keys(): - tensor_stat = data["tensor_stat"] - if len(tensor_stat) != Const.TENSOR_STAT_LEN or len(tensor_stat[0]) != len(tensor_stat[1]): - logger.warning("Some bad data in async dump") - else: - tensor_stat_index, tensor_stat_data = tensor_stat[0], tensor_stat[1] - if hasattr(tensor_stat_data, "device") and tensor_stat_data.device != Const.CPU_LOWERCASE: - tensor_stat_data = tensor_stat_data.cpu() - for index, stat in zip(tensor_stat_index, tensor_stat_data): - data.update({index: stat.item()}) - del data["tensor_stat"] - else: - for key in data.keys(): - self.process_stat_data_recursive(data[key], depth + 1) - elif isinstance(data, (list, tuple)): - for i in data: - self.process_stat_data_recursive(i, depth + 1) \ No newline at end of file + with lock: + # 在写 JSON 前，统一获取统计值 + stat_result = self.flush_stat_stack() + # 遍历 cache_data，将占位符替换为最终统计值 + if stat_result: + self._replace_stat_placeholders(self.cache_data, stat_result) + if self.cache_debug: + self._replace_stat_placeholders(self.cache_debug, stat_result) + if self.cache_data: + self.write_data_json(self.dump_file_path) + if self.cache_stack: + self.write_stack_info_json(self.stack_file_path) + if self.cache_construct: + self.write_construct_info_json(self.construct_file_path) + if self.cache_debug: + self.write_debug_info_json(self.debug_file_path) + diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/constant.py b/debug/accuracy_tools/msprobe/core/grad_probe/constant.py index 22a8b6c13411b68a6566d0686062f8c74cb27196..5d9c72a6f2d60203b0d9ba716e867e39ee22d807 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/constant.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/constant.py @@ -31,6 +31,7 @@ class GradConst: STEP = "step" BOUNDS = "bounds" OUTPUT_PATH = "output_path" + TIME_STAMP = "time_stamp" # level const LEVEL = "level" @@ -51,7 +52,7 @@ class GradConst: BOUNDS_MINIMUM = -2**63 BOUNDS_MAXIMUM = 2**63 - 1 - # file safty + # file safety DATA_DIR_AUTHORITY = 0o750 DATA_FILE_AUTHORITY = 0o640 DIRECTORY_LENGTH = 4096 diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 4f2b25bd28dfe330a8716695278ab8c64222c4b6..f50fc0f4e381db0e4069ef99b5c70b593f1580d0 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -112,7 +112,7 @@ class GradComparator: result.append([key] + value) result_csv_path = os.path.join(output_dir, "similarities.csv") if os.path.exists(result_csv_path): - logger.warning(f"{result_csv_path} will be recoverd") + logger.warning(f"{result_csv_path} will be deleted") remove_path(result_csv_path) write_csv(result, result_csv_path) @@ -121,7 +121,7 @@ class GradComparator: similarities = {} logger.info(f"{len(steps)} steps will be compared") grad_weight_order = cls._get_grad_weight_order(path1, path2) - for step in tqdm(steps, desc="culculate similarities (by step)"): + for step in tqdm(steps, desc="calculate similarities (by step)"): grad_files = cls._get_matched_grad_files(path1, path2, step) same_count_summary = 0 total_count_summary = 0 diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py index de3e4156acc74f135120e06116b5894a0e9ed09e..468367a54a8bf4926edd5a8f25cefaa5890ec40c 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py @@ -82,7 +82,7 @@ class ListCache(list): if len(self) == 0: return if not self._output_file: - logger.warning("dumpfile path is not setted") + logger.warning("dumpfile path is not set.") write_csv(self, self._output_file) logger.info(f"write {len(self)} items to {self._output_file}.") self.clear() diff --git a/debug/accuracy_tools/msprobe/core/overflow_check/abnormal_scene.py b/debug/accuracy_tools/msprobe/core/overflow_check/abnormal_scene.py index 54dae2576e48b7ad75df97fa046e6e90bbd144c2..0e0c50cc6aa0cf93f963a699ee36c13d888ec320 100644 --- a/debug/accuracy_tools/msprobe/core/overflow_check/abnormal_scene.py +++ b/debug/accuracy_tools/msprobe/core/overflow_check/abnormal_scene.py @@ -20,6 +20,7 @@ import numpy as np from msprobe.core.overflow_check.api_info import APIInfo from msprobe.core.overflow_check.level import OverflowLevel from msprobe.core.overflow_check.utils import has_nan_inf +from msprobe.core.common.decorator import recursion_depth_decorator class AnomalyScene: @@ -35,6 +36,7 @@ class AnomalyScene: raise NotImplementedError @staticmethod + @recursion_depth_decorator("AbnormalScene: AnomalyScene._has_anomaly") def _has_anomaly(data: Union[Dict, Any]) -> bool: """检查张量是否包含异常值""" if isinstance(data, dict): diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/unittest/__init__.py b/debug/accuracy_tools/msprobe/core/single_save/__init__.py similarity index 100% rename from debug/accuracy_tools/msprobe/pytorch/monitor/unittest/__init__.py rename to debug/accuracy_tools/msprobe/core/single_save/__init__.py diff --git a/debug/accuracy_tools/msprobe/core/single_save/single_comparator.py b/debug/accuracy_tools/msprobe/core/single_save/single_comparator.py new file mode 100644 index 0000000000000000000000000000000000000000..3d60422fca7c3186923ecc069e651ce3268a29b4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/single_save/single_comparator.py @@ -0,0 +1,243 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import multiprocessing +from dataclasses import dataclass + +import numpy as np +import pandas as pd +from tqdm import tqdm + +from msprobe.core.common.file_utils import check_file_or_directory_path, create_directory, save_excel +from msprobe.core.common.log import logger + + +@dataclass +class CompareResult: + max_abs_error: float + max_relative_error: float + same_percentage: float + first_mismatch_index: int + percentage_within_thousandth: float + percentage_within_hundredth: float + + +class SingleComparator: + result_header = [ + 'step', + 'rank', + 'micro_step', + 'id', + 'shape1', + 'shape2', + '相同元素百分比(%)', + '首个不匹配元素索引', + '最大绝对误差', + '最大相对误差', + '误差在千分之一内元素占比(%)', + '误差在百分之一内元素占比(%)' + ] + + @classmethod + def compare(cls, dir1, dir2, output_path="./msprobe_compare_output", num_processes=8): + data_dir1 = os.path.join(dir1, "data") + data_dir2 = os.path.join(dir2, "data") + check_file_or_directory_path(data_dir1, isdir=True) + check_file_or_directory_path(data_dir2, isdir=True) + # 确保输出目录存在，如果不存在则创建 + if not os.path.exists(output_path): + create_directory(output_path) + cls.compare_data(data_dir1, data_dir2, output_path, num_processes) + + @classmethod + def compare_arrays(cls, array1, array2) -> CompareResult: + """ + 比较两个NumPy数组，计算最大绝对误差、最大相对误差和相同元素的百分比 + """ + # 计算每个维度上的最小尺寸 + min_shape = [min(s1, s2) for s1, s2 in zip(array1.shape, array2.shape)] + # 截取数组到相同的形状 + sliced_array1 = array1[tuple(slice(0, s) for s in min_shape)] + sliced_array2 = array2[tuple(slice(0, s) for s in min_shape)] + + abs_error = np.abs(sliced_array1 - sliced_array2) + max_abs_error = np.max(abs_error) + + # 计算相对误差，处理分母为零的情况 + with np.errstate(divide='ignore', invalid='ignore'): + relative_error = np.abs(sliced_array1 - sliced_array2) / \ + np.maximum(np.abs(sliced_array1), np.abs(sliced_array2)) + relative_error = np.nan_to_num(relative_error) + max_relative_error = np.max(relative_error) + + same_elements = np.sum(sliced_array1 == sliced_array2) + total_elements = sliced_array1.size + same_percentage = (same_elements / total_elements) * 100 + + # 展平数组 + flat_array1 = sliced_array1.flatten() + flat_array2 = sliced_array2.flatten() + + # 计算从第几个元素开始对不上 + mismatch_indices = np.nonzero(flat_array1 != flat_array2)[0] + first_mismatch_index = mismatch_indices[0] if mismatch_indices.size > 0 else None + + # 计算误差在千分之一内的元素占比 + threshold = 0.001 * np.maximum(np.abs(sliced_array1), np.abs(sliced_array2)) + error_within_thousandth = np.sum(abs_error <= threshold) + percentage_within_thousandth = (error_within_thousandth / total_elements) * 100 + + # 计算误差在百分之一内的元素占比 + threshold = 0.01 * np.maximum(np.abs(sliced_array1), np.abs(sliced_array2)) + error_within_hundredth = np.sum(abs_error <= threshold) + percentage_within_hundredth = (error_within_hundredth / total_elements) * 100 + + return CompareResult( + max_abs_error, + max_relative_error, + same_percentage, + first_mismatch_index, + percentage_within_thousandth, + percentage_within_hundredth + ) + + @classmethod + def get_steps(cls, tag_path): + for step_folder in os.listdir(tag_path): + if step_folder.startswith('step'): + try: + step = int(step_folder[4:]) + except Exception as e: + raise RuntimeError(f"parse step number error") from e + yield step, os.path.join(tag_path, step_folder) + + @classmethod + def get_ranks(cls, step_path): + for rank_folder in os.listdir(step_path): + if rank_folder.startswith('rank'): + try: + rank = int(rank_folder[4:]) + except Exception as e: + raise RuntimeError(f"parse rank number error") from e + yield rank, os.path.join(step_path, rank_folder) + + @classmethod + def get_micro_steps(cls, rank_path): + for micro_step_folder in os.listdir(rank_path): + if micro_step_folder.startswith('micro_step'): + try: + micro_step = int(micro_step_folder[10:]) + except Exception as e: + raise RuntimeError(f"parse nicro_step number error") from e + yield micro_step, os.path.join(rank_path, micro_step_folder) + else: + yield 0, rank_path + + @classmethod + def get_arrays(cls, micro_step_path): + for file in os.listdir(micro_step_path): + if file.endswith('.npy'): + try: + parts = file.rsplit('.', 2) + if len(parts) > 1 and parts[-2].isdigit(): + array_id = int(parts[-2]) + else: + array_id = 0 + except ValueError: + array_id = 0 + yield array_id, os.path.join(micro_step_path, file) + + @classmethod + def get_array_paths(cls, dir_path): + """ + 获取目录中所有符合结构的NumPy数组文件路径 + """ + array_paths = {} + if not os.path.exists(dir_path): + return array_paths + for tag in os.listdir(dir_path): + tag_path = os.path.join(dir_path, tag) + if not os.path.isdir(tag_path): + continue + for step, step_path in cls.get_steps(tag_path): + for rank, rank_path in cls.get_ranks(step_path): + for micro_step, micro_step_path in cls.get_micro_steps(rank_path): + for array_id, array_path in cls.get_arrays(micro_step_path): + array_paths.setdefault(tag, []).append((step, rank, micro_step, array_id, array_path)) + return array_paths + + @classmethod + def compare_single_tag(cls, tag, array_paths1, array_paths2, output_dir): + try: + data = [] + paths1 = array_paths1.get(tag, []) + paths2 = array_paths2.get(tag, []) + path_dict1 = {(step, rank, micro_step, array_id): path for step, rank, micro_step, array_id, path in paths1} + path_dict2 = {(step, rank, micro_step, array_id): path for step, rank, micro_step, array_id, path in paths2} + common_keys = set(path_dict1.keys()) & set(path_dict2.keys()) + for key in common_keys: + try: + array1 = np.load(path_dict1[key]) + array2 = np.load(path_dict2[key]) + result = cls.compare_arrays(array1, array2) + step, rank, micro_step, array_id = key + data.append([ + step, rank, micro_step, array_id, + list(array1.shape), list(array2.shape), + result.same_percentage, + result.first_mismatch_index, + result.max_abs_error, + result.max_relative_error, + result.percentage_within_thousandth, + result.percentage_within_hundredth + ]) + except Exception as e: + logger.error(f"Error comparing {path_dict1[key]} and {path_dict2[key]}: {e}") + + df = pd.DataFrame(data, columns=SingleComparator.result_header) + df = df.sort_values(by=['step', 'rank', 'micro_step', 'id']) + # 构建输出文件的完整路径 + output_file_path = os.path.join(output_dir, f'{tag}.xlsx') + save_excel(output_file_path, df) + except Exception as e: + logger.error(f"Error processing tag {tag}: {e}") + + @classmethod + def compare_data(cls, dir1, dir2, output_dir, num_processes=8): + """ + 比较两个目录中的NumPy数组文件，并将结果保存到指定目录的Excel文件中 + """ + + array_paths1 = cls.get_array_paths(dir1) + array_paths2 = cls.get_array_paths(dir2) + + all_tags = set(array_paths1.keys()) | set(array_paths2.keys()) + + with multiprocessing.Pool(processes=num_processes) as pool: + args = [(tag, array_paths1, array_paths2, output_dir) for tag in all_tags] + try: + results = pool.starmap_async(cls.compare_single_tag, args) + with tqdm(total=len(all_tags), desc="Processing data") as pbar: + while not results.ready(): + pbar.n = len(all_tags) - results._number_left + pbar.refresh() + results.wait() + results.get() + except Exception as e: + logger.error(f"Multiprocessing error: {e}") + finally: + pool.close() + pool.join() diff --git a/debug/accuracy_tools/msprobe/core/single_save/single_saver.py b/debug/accuracy_tools/msprobe/core/single_save/single_saver.py new file mode 100644 index 0000000000000000000000000000000000000000..5fea8f0baba7d9b737c4ba5880d00bb3f6233bb1 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/single_save/single_saver.py @@ -0,0 +1,157 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from msprobe.core.common.file_utils import create_directory, save_json +from msprobe.core.common.const import Const +from msprobe.core.common.framework_adapter import FmkAdp +from msprobe.core.common.log import logger + + +support_nested_data_type = (list, tuple, dict) + + +class SingleSave: + _instance = None + + def __new__(cls, dump_path, fmk=Const.PT_FRAMEWORK): + if cls._instance is None: + cls._instance = super(SingleSave, cls).__new__(cls) + FmkAdp.set_fmk(fmk) + create_directory(dump_path) + + cls._instance.dump_path = dump_path + cls._instance.rank = FmkAdp.get_rank_id() + cls._instance.step_count = 0 + cls._instance.cache_dict = {} + return cls._instance + + @staticmethod + def _analyze_tensor_data(data, data_name=None, save_dir=None): + ''' + data: Tensor + return: + result_data: with keys {"max", "min", "mean", "norm", "shape"} + ''' + result_data = {} + result_data["max"] = FmkAdp.tensor_max(data) + result_data["min"] = FmkAdp.tensor_min(data) + result_data["mean"] = FmkAdp.tensor_mean(data) + result_data["norm"] = FmkAdp.tensor_norm(data) + result_data["shape"] = list(data.shape) + if save_dir is not None and data_name is not None: + real_save_path = os.path.join(save_dir, data_name + ".npy") + FmkAdp.save_tensor(data, real_save_path) + return result_data + + @classmethod + def save_config(cls, data): + dump_file = os.path.join(cls._instance.dump_path, 'configurations.json') + save_json(dump_file, data, indent=4) + + @classmethod + def save_ex(cls, data, micro_batch=None): + ''' + data: dict{str: Union[Tensor, tuple, list]} + + return: void + ''' + + instance = cls._instance + + if not isinstance(data, dict): + logger.warning("SingleSave data type not valid, " + "should be dict. " + "Skip current save process.") + return + for key, value in data.items(): + if not isinstance(key, str): + logger.warning("key should be string when save data") + continue + if not isinstance(value, support_nested_data_type) and not FmkAdp.is_tensor(value): + logger.warning(f"value should be {support_nested_data_type} or Tensor when save data") + continue + real_dump_dir = os.path.join( + instance.dump_path, + "data", + key, + f"step{instance.step_count}", + f"rank{instance.rank}") + if micro_batch is not None: + real_dump_dir = os.path.join(real_dump_dir, f"micro_step{micro_batch}") + create_directory(real_dump_dir) + + if FmkAdp.is_tensor(value): + result = cls._analyze_tensor_data(value, key, real_dump_dir) + elif isinstance(value, (tuple, list)): + result = cls._analyze_list_tuple_data(value, key, real_dump_dir) + elif isinstance(value, dict): + result = cls._analyze_dict_data(value, key, real_dump_dir) + + result_json = {"data": result} + json_path = os.path.join(real_dump_dir, key + ".json") + save_json(json_path, result_json, indent=4) + + + @classmethod + def step(cls): + instance = cls._instance + for key, value in instance.cache_dict.items(): + if not value["have_micro_batch"]: + cls.save_ex({key: value["data"][0]}) + else: + for i, data in enumerate(value["data"]): + cls.save_ex({key: data}, micro_batch=i) + instance.cache_dict = {} + instance.step_count += 1 + + @classmethod + def save(cls, data): + instance = cls._instance + if not isinstance(data, dict): + logger.warning("SingleSave data type not valid, " + "should be dict. " + "Skip current save process.") + return + for key, value in data.items(): + if key not in instance.cache_dict: + instance.cache_dict[key] = { + "have_micro_batch": False, + "data": [value] + } + else: + instance.cache_dict[key]["have_micro_batch"] = True + instance.cache_dict[key]["data"].append(value) + + @classmethod + def _analyze_list_tuple_data(cls, data, data_name=None, save_dir=None): + lst = [] + for index, element in enumerate(data): + if not FmkAdp.is_tensor(element): + raise TypeError(f"SingleSave: Unsupported type: {type(element)}") + element_name = data_name + "." + str(index) + lst.append(cls._analyze_tensor_data(element, element_name, save_dir)) + return lst + + @classmethod + def _analyze_dict_data(cls, data, data_name=None, save_dir=None): + result_data = {} + for key, value in data.items(): + if not FmkAdp.is_tensor(value): + raise TypeError(f"SingleSave: Unsupported type: {type(value)}") + key_name = data_name + "." + str(key) + result_data[key] = cls._analyze_tensor_data(value, key_name, save_dir) + return result_data diff --git a/debug/accuracy_tools/msprobe/docs/01.installation.md b/debug/accuracy_tools/msprobe/docs/01.installation.md index 530783e87d0bdadd51856cb1ae08160cb081da80..b5077228919c713c5e7910703678339c0b809326 100644 --- a/debug/accuracy_tools/msprobe/docs/01.installation.md +++ b/debug/accuracy_tools/msprobe/docs/01.installation.md @@ -16,6 +16,8 @@ pip install mindstudio-probe |版本|发布日期|支持 PyTorch 版本|支持 MindSpore 版本|下载链接|校验码| |:--:|:--:|:--:|:--:|:--:|:--:| +|8.0.0|2025.5.07|1.11/2.0/2.1/2.2|2.4.0/2.5.0/2.6.0|[mindstudio_probe-8.0.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/8.0/mindstudio_probe-8.0.0-py3-none-any.whl)|6810eade7ae99e3b24657d5cab251119882decd791aa76a7aeeb94dea767daec| +|1.3.0|2025.4.17|1.11/2.0/2.1/2.2|2.4.0/2.5.0/2.6.0|[mindstudio_probe-1.3.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.3/mindstudio_probe-1.3.0-py3-none-any.whl)|85dbc5518b5c23d29c67d7b85d662517d0318352f372891f8d91e73e71b439c3| |1.2.2|2025.3.03|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.2/mindstudio_probe-1.2.2-py3-none-any.whl)|961411bb460d327ea51d6ca4d0c8e8c5565f07c0852d7b8592b781ca35b87212| |1.2.1|2025.2.07|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.2/mindstudio_probe-1.2.1-py3-none-any.whl)|b64b342118558e0339b39237f88a49b93fd24551b0cb202c872fbfef4260c86b| |1.2.0|2025.1.13|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.2/mindstudio_probe-1.2.0-py3-none-any.whl)|1e3aeea1706112f6ee52fd1165037936bb209138f0b9ec42ea21e2c1c8942cdc| @@ -52,7 +54,7 @@ pip install ./mindstudio_probe*.whl |参数|说明|是否必选| |--|--|:--:| -|--include-mod|指定可选模块，可取值`adump`，表示在编whl包时加入adump模块。默认未配置该参数，表示编基础包。
• adump模块用于MindSpore静态图场景L2级别的dump。
• 仅MindSpore 2.5.0及以上版本支持adump模块。
• 若使用源码安装，编译环境需支持GCC 7或以上版本，和CMAKE 3.14或以上版本。
• 生成的whl包仅限编译时使用的python版本和处理器架构可用。|否| +|--include-mod|指定可选模块，可取值`adump`，表示在编whl包时加入adump模块。默认未配置该参数，表示编基础包。
• adump模块用于MindSpore静态图场景L2级别的dump。
• 仅MindSpore 2.5.0及以上版本支持adump模块。
• 若使用源码安装，编译环境需支持GCC 7.5或以上版本，和CMAKE 3.14或以上版本。
• 生成的whl包仅限编译时使用的python版本和处理器架构可用。|否| # 特性变更说明 @@ -80,8 +82,6 @@ pip install ./mindstudio_probe*.whl ## 1.1.1 -## 1.1.1 - 【数据采集】 - dump 支持 processgroup、namedtuple、slice 等数据类型 diff --git a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md index 188123c0efcd34cc02f39136407ad0b68d9354aa..6da5a4820c3917d7e743875c277b984594a09e8b 100644 --- a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md +++ b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md @@ -10,26 +10,25 @@ ### 1.1 通用配置 -| 参数 | 解释 | 是否必选 | -| ----------------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | -| task | dump 的任务类型，str 类型。可选参数：
"statistics"：仅采集统计信息，默认值；
"tensor"：采集统计信息和完全复刻整网的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持，采集数据时勿选；
"overflow_check"：溢出检测；
"free_benchmark"：无标杆比对，不支持 MSAdapter 场景；
"grad_probe"：梯度监控， 不支持 MSAdapter 场景；
"structure"：仅采集模型结构以及调用栈信息，不采集具体数据。
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)，
[1.7 task 配置为 grad_probe](#17-task-配置为-grad_probe)。
**配置示例**："task": "tensor"。 | 否 | -| dump_path | 设置 dump 数据目录路径，str 类型。
**配置示例**："dump_path": "./dump_path"。 | 是 | -| rank | 指定对某张卡上的数据进行采集，list[Union[int, str]] 类型，默认未配置（表示采集所有卡的数据），应配置元素为 ≥0 的整数或类似"4-6"的字符串，且须配置实际可用的 Rank ID。
PyTorch 场景: Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，dump 数据为空。
MindSpore 场景：所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
注意，单卡训练时，rank必须为[]，即空列表，不能指定rank。
**配置示例**："rank": [1, "4-6"]。 | 否 | -| step | 指定采集某个 step 的数据，list[Union[int, str]] 类型。默认未配置，表示采集所有 step 数据。采集特定 step 时，须指定为训练脚本中存在的 step，可逐个配置，也可以指定范围。
**配置示例**："step": [0, 1 , 2, "4-6"]。 | 否 | -| level | dump 级别，str 类型，根据不同级别采集不同数据。可选参数：
"L0"：dump 模块级精度数据，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持，使用背景详见 [1.1.1 模块级精度数据 dump 说明](#111-模块级精度数据-dump-说明)；
"L1"：dump API 级精度数据，默认值，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持；
"L2"：dump kernel 级精度数据，PyTorch 场景详细介绍见 [PyTorch 场景的 kernel dump 说明](./04.kernel_dump_PyTorch.md)；MindSpore 动态图场景详细介绍见 [MindSpore 动态图场景的 kernel dump 说明](./28.kernel_dump_MindSpore.md)；MindSpore 静态图场景详细介绍见《MindSpore 场景的数据采集》中的 ["**8.1 静态图场景**"](./06.data_dump_MindSpore.md#81-静态图场景)小节；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持。
"debug"：单点保存功能，细节详见[单点保存工具 README](./28.debugger_save_instruction.md)
**配置示例**："level": "L1"。 | 否 | -| enable_dataloader | 自动控制开关，bool 类型，仅 PyTorch 场景支持。可选参数 true（开启）或 false（关闭），默认为 false。配置为 true 后自动识别 step 参数指定的迭代，并在该迭代执行完成后退出训练，此时 start、stop 和 step 函数可不配置，开启该开关要求训练脚本是通过 torch.utils.data.dataloader 方式加载数据。仅支持 PyTorch 单卡训练使用，分布式训练场景下存在数据 dump 不全问题。 **这个特性下个版本将被废弃** | 否 | -| async_dump | 异步 dump 开关，bool 类型。可选参数 true（开启）或 false（关闭），默认为 false。配置为 true 后开启异步 dump，即采集的精度数据会在当前 step 训练结束后统一落盘，训练过程中工具不触发同步操作。由于使用该模式有**显存溢出**的风险，当 task 配置为 tensor 时，即真实数据的异步dump模式，必须配置 [list](#13-task-配置为-tensor) 参数，指定需要 dump 的 tensor 。该模式暂不支持复数类型 tensor
的统计量计算。 | 否 | +| 参数 | 解释 | 是否必选 | +| ----------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | +| task | dump 的任务类型，str 类型。可选参数：
"statistics"：仅采集统计信息，默认值；
"tensor"：采集统计信息和完全复刻整网的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持，采集数据时勿选；
"overflow_check"：溢出检测；
"free_benchmark"：无标杆比对，不支持 MSAdapter 场景；
"grad_probe"：梯度监控， 不支持 MSAdapter 场景；
"structure"：仅采集模型结构以及调用栈信息，不采集具体数据。
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)，
[1.7 task 配置为 grad_probe](#17-task-配置为-grad_probe)，
[1.8 task 配置为 structure](#18-task-配置为-structure)。
**配置示例**："task": "tensor"。 | 否 | +| dump_path | 设置 dump 数据目录路径，str 类型。
**配置示例**："dump_path": "./dump_path"。 | 是 | +| rank | 指定对某张卡上的数据进行采集，list[Union[int, str]] 类型，默认未配置（表示采集所有卡的数据），应配置元素为 ≥0 的整数或类似"4-6"的字符串，且须配置实际可用的 Rank ID。
PyTorch 场景: Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，dump 数据为空。
MindSpore 场景：所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。静态图 L0 级别 dump 暂不支持指定rank。
注意，单卡训练时，rank必须为[]，即空列表，不能指定rank。
**配置示例**："rank": [1, "4-6"]。 | 否 | +| step | 指定采集某个 step 的数据，list[Union[int, str]] 类型。默认未配置，表示采集所有 step 数据。采集特定 step 时，须指定为训练脚本中存在的 step，可逐个配置，也可以指定范围。
**配置示例**："step": [0, 1 , 2, "4-6"]。 | 否 | +| level | dump 级别，str 类型，根据不同级别采集不同数据。可选参数：
"L0"：dump 模块级精度数据，使用背景详见 [1.1.1 模块级精度数据 dump 说明](#111-模块级精度数据-dump-说明)。
"L1"：dump API 级精度数据，默认值，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持。
"L2"：dump kernel 级精度数据，PyTorch 场景详细介绍见 [PyTorch 场景的 kernel dump 说明](./04.kernel_dump_PyTorch.md)；MindSpore 动态图场景详细介绍见 [MindSpore 动态图场景的 kernel dump 说明](./28.kernel_dump_MindSpore.md)；MindSpore 静态图场景详细介绍见《MindSpore 场景的数据采集》中的 ["**8.1 静态图场景**"](./06.data_dump_MindSpore.md#81-静态图场景)小节。
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持。
"debug"：单点保存功能，详见[单点保存工具](./28.debugger_save_instruction.md)。
**配置示例**："level": "L1"。 | 否 | +| enable_dataloader | 自动控制开关，bool 类型，仅 PyTorch 场景支持。可选参数 true（开启）或 false（关闭），默认为 false。配置为 true 后自动识别 step 参数指定的迭代，并在该迭代执行完成后退出训练，此时 start、stop 和 step 函数可不配置，开启该开关要求训练脚本是通过 torch.utils.data.dataloader 方式加载数据。仅支持 PyTorch 单卡训练使用，分布式训练场景下存在数据 dump 不全问题。 **这个特性下个版本将被废弃** | 否 | +| async_dump | 异步 dump 开关，bool 类型。可选参数 true（开启）或 false（关闭），默认为 false。配置为 true 后开启异步 dump，即采集的精度数据会在当前 step 训练结束后统一落盘，训练过程中工具不触发同步操作。由于使用该模式有**显存溢出**的风险，当 task 配置为 tensor 时，即真实数据的异步dump模式，需配置 [list](#13-task-配置为-tensor) 参数（mindspore静态图O0/O1场景无此问题，不强制），指定需要 dump 的 tensor 。该模式暂不支持复数类型 tensor
的统计量计算。 | 否 | #### 1.1.1 模块级精度数据 dump 说明 -仅 PyTorch、MSAdapter以及 MindSpore 动态图场景支持。 - 大模型场景下，通常不是简单的利用自动迁移能力实现从 GPU 到 NPU 的训练脚本迁移，而是会对 NPU 网络进行一系列针对性的适配，因此，常常会造成迁移后的 NPU 模型存在部分子结构不能与 GPU 原始模型完全对应。模型结构不一致导致 API 调用类型及数量不一致，若直接按照 API 粒度进行精度数据 dump 和比对，则无法完全比对所有的 API。 本小节介绍的功能是对模型中的大粒度模块进行数据 dump，使其比对时，对于无法以 API 粒度比对的模块可以直接以模块粒度进行比对。 模块指的是继承 nn.Module 类（PyTorch 与 MSAdapter 场景）或 nn.Cell 类（MindSpore 场景）的子类，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump 数据时以模块为粒度进行 dump。 +特别地，在PyTorch场景中，为了规避BackwardHook函数的输出不能进行原地操作的框架限制，工具使用了`torch._C._autograd._set_creation_meta`接口对BackwardHook函数的输出张量进行属性重置，这可能会造成dump数据中缺少原地操作模块（nn.ReLU(inplace=True)及其上一个模块的反向数据。 ### 1.2 task 配置为 statistics @@ -44,15 +43,27 @@
配置示例："list": ["Module.module.language_model.encoder.layers.0.mlp.ParallelMlp.forward.0"], 或 "list": ["Cell.network_with_loss.language_model.encoder.layers.0.mlp.ParallelMlp.forward.0"] PyTorch、MSAdapter 以及 MindSpore 动态图场景指定某一类 API，dump 某一类的 API 级别输入输出数据。
配置示例："list": ["relu"]。
PyTorch、MSAdapter 以及 MindSpore 动态图场景在level为 mix 级别时, 会dump名称中包含list中配置的字符串的API数据，还会将名称中包含list中配置的字符串的模块进行展开dump （dump该模块从执行开始到执行结束期间的所有数据）。 MindSpore 静态图场景配置 kernel_name，可以是算子的名称列表，也可以指定算子类型（jit_level=O2 时不支持），还可以配置算子名称的正则表达式（当字符串符合“name-regex(xxx)”格式时，后台则会将其作为正则表达式。
配置示例：list: ["name-regex(Default/.+)"]
可匹配算子名称以“Default/”开头的所有算子。 - data_modedump 数据过滤，str 类型。否 + tensor_list自定义采集真实数据的算子列表，list[str] 类型，默认未配置。包含以下配置方法：否 + PyTorch、MSAdapter 以及 MindSpore 动态图场景指定某一类 API 或模块，即会 dump 这一类 API 或模块输入输出的统计量信息和完整的 tensor 数据。
配置示例："tensor_list": ["relu"]。
PyTorch、MSAdapter 以及 MindSpore 动态图场景目前只支持level配置为 L0, L1 和 mix 级别。
MindSpore 静态图场景不支持。 + data_modedump 数据过滤，str 类型。否 + device控制统计值计算所用的设备，可选值["device", "host"]，默认"host"。使用device计算会比host有性能加速，只支持min/max/avg/l2norm统计量。支持 MindSpore静态图 O0/O1 场景。否 + precision控制统计值计算所用精度，可选值["high", "low"]，默认值为"high"。选择"high"时，avg/l2norm统计量使用float32进行计算，会增加device内存占用，精度更高；为"low"时使用与原始数据相同的类型进行计算，device内存占用较少，但在处理较大数值时可能会导致统计量溢出。支持 MindSpore静态图 O0/O1 场景。否 PyTorch、MSAdapter 以及 MindSpore 动态图场景：支持"all"、"forward"、"backward"、"input"和"output"，除"all"外，其余参数可以自由组合。默认为["all"]，即保存所有 dump 的数据。
配置示例："data_mode": ["backward"] （仅保存反向数据）或 "data_mode": ["forward", "input"]（仅保存前向的输入数据）。 - MindSpore 静态图场景：仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。
配置示例："data_mode": ["all"]。 + MindSpore 静态图场景：L0 级别 dump 仅支持"all"、"forward"和"backward"参数；L2 级别 dump 仅支持"all"、"input"和"output"参数。且各参数只能单独配置，不支持自由组合。
配置示例："data_mode": ["all"]。 summary_mode控制 dump 文件输出的模式，str 类型，支持 PyTorch、MSAdapter、MindSpore 动态图以及 MindSpore 静态图 jit_level=O2 场景。否 PyTorch、MSAdapter 以及 MindSpore 动态图场景：可选参数为
md5：dump 输出包含 CRC-32 值以及 API 统计信息的 dump.json 文件，用于验证数据的完整性；
statistics：dump 仅输出包含 API 统计信息的 dump.json 文件，默认值。
配置示例："summary_mode": "md5"。 MindSpore 静态图 jit_level=O2 场景：支持上述配置的同时额外支持配置统计项列表，可选统计项为max、min、mean、l2norm，可从中任意选取组合搭配。其中mean、l2norm的结果为float数据格式。
配置示例："summary_mode": ["max", "min"]。 -**说明**："summary_mode" 配置为 "md5" 时，所使用的校验算法为 CRC-32 算法。 +**说明**： + + +1. "summary_mode" 配置为 "md5" 时，所使用的校验算法为 CRC-32 算法。 + +**示例**： + - [PyTorch场景](03.config_examples.md#11-task-配置为-statistics) + - [MindSpore静态图场景](03.config_examples.md#21-task-配置为-statistics) + - [MindSpore动态图场景](03.config_examples.md#31-task-配置为-statistics) ### 1.3 task 配置为 tensor @@ -62,12 +73,21 @@ | list | 与[ 1.2 task 配置为 statistics ](#12-task-配置为-statistics)中的解释相同。 | 否 | | data_mode | 与[ 1.2 task 配置为 statistics ](#12-task-配置为-statistics)中的解释相同 | 否 | | file_format | tensor 数据的保存格式，str 类型，仅支持 MindSpore 静态图场景的 L2 级别配置该字段，其他场景不生效。可选参数：
"bin"：dump 的 tensor 文件为二进制格式；
"npy"：dump 的 tensor 文件后缀为 .npy，默认值。 | 否 | +| summary_mode | 控制 dump 文件输出的模式，str 类型，支持 PyTorch、MSAdapter、MindSpore 动态图。可选参数：
md5：dump 输出包含 CRC-32 值以及 API 统计信息的 dump.json 文件，用于验证数据的完整性；
statistics：dump 仅输出包含 API 统计信息的 dump.json 文件，默认值。| 否 | | online_run_uta | 在线预检模式开关，bool 类型，可选参数 true（开启）、false（关闭），默认未配置，表示关闭。配置为 true 表示开启在线预检。| 否 | | nfs_patha | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。仅在 online_run_ut 字段配置为 true 时生效，配置该参数后 host 和 port 不生效。 | 否 | | hosta | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的局域网 IP 地址。仅在 online_run_ut 字段配置为 true 时生效，局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | | porta | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的端口号。仅在 online_run_ut 字段配置为 true 时生效，局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。| 否 | -**a**：online_run_ut、nfs_path、host、port 等字段仅在线预检场景 NPU 机器生效。 +**说明**： + +1. online_run_ut、nfs_path、host、port 等字段仅在线预检场景 NPU 机器生效。 + +**示例**： + - [PyTorch场景](03.config_examples.md#12-task-配置为-tensor) + - [MindSpore静态图场景](03.config_examples.md#22-task-配置为-tensor) + - [MindSpore动态图场景](03.config_examples.md#32-task-配置为-tensor) + ### 1.4 task 配置为 run_ut @@ -82,9 +102,28 @@ | portb | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机可用端口。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。仅在 is_online 字段配置为 true 时生效。| 否 | | rank_listb | 指定在线预检的 Rank ID，默认值为 [0]，list[int] 类型，应配置为大于等于 0 的整数，且须根据实际卡的 Rank ID 配置，若所配置的值大于实际训练所运行的卡的 Rank ID，则在线预检输出数据为空。GPU 和 NPU 须配置一致。仅在 is_online 字段配置为 true 时生效。 | 否 | -**a**：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 +**说明**： + +1. white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 + +2. is_online、nfs_path、host、port、rank_list 等字段仅在线预检场景 GPU 机器生效。 -**b**：is_online、nfs_path、host、port、rank_list 等字段仅在线预检场景 GPU 机器生效。 +**示例**： +```json +{ + "task": "run_ut", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + + "run_ut": { + "white_list": [], + "black_list": [], + "error_data_path": "./" + } +} +``` ### 1.5 task 配置为 overflow_check @@ -95,6 +134,11 @@ PyTorch、MSAdapter 以及 MindSpore 动态图场景下，"level"须为"L0"或"L | overflow_nums | 最大溢出次数，int 类型，默认为 1，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持。表示第 N 次溢出后，不再进行溢出检测。过程中检测到溢出 API 对应的 输入输出 数据均 dump。
**配置示例**："overflow_nums": 3。配置为 -1 时，表示持续检测溢出直到训练结束。 | 否 | | check_mode | 溢出类型，str 类型，仅 MindSpore v2.3.0 以下版本的静态图场景支持，可选参数：
"aicore"：开启 AI Core 的溢出检测；
"atomic"：开启 Atomic 的溢出检测；
"all"：开启算子的溢出检测，默认值。
**配置示例**："check_mode": "all"。 | 否 | +**示例**： + - [PyTorch场景](03.config_examples.md#14-task-配置为-overflow_check) + - [MindSpore静态图场景](03.config_examples.md#23-task-配置为-overflow_check) + - [MindSpore动态图场景](03.config_examples.md#33-task-配置为-overflow_check) + ### 1.6 task 配置为 free_benchmark 仅 PyTorch 与 MindSpore 动态图场景支持，且"level"为"L1"。 @@ -121,6 +165,10 @@ PyTorch、MSAdapter 以及 MindSpore 动态图场景下，"level"须为"L0"或"L max_sample每个算子预热的采样次数的最大阈值（仅 PyTorch 场景支持），int 类型，默认值为 20。须配置 "if_preheat": "true"。否 +**示例**： + - [PyTorch场景](03.config_examples.md#15-task-配置为-free_benchmark) + - [MindSpore动态图场景](03.config_examples.md#34-task-配置为-free_benchmark) + #### 1.6.1 无标杆比对数据存盘格式 无标杆比对在 dump_path 目录下输出结果文件 `free_benchmark.csv`，如下示例： @@ -164,5 +212,15 @@ PyTorch、MSAdapter 以及 MindSpore 动态图场景下，"level"须为"L0"或"L | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | - intervals就是根据值分布bounds划分出的区间。 - MindSpore静态图模式下，L0级别中暂不支持"MD5" +**说明**： + +1. intervals就是根据值分布bounds划分出的区间。 +2. MindSpore静态图模式下，L0级别中暂不支持"MD5" + +### 1.8 task 配置为 structure +structure 模式仅采集模型结构，无其他特殊配置。 + +**示例**： + - [PyTorch场景](03.config_examples.md#16-task-配置为-structure) + - [MindSpore动态图场景](03.config_examples.md#35-task-配置为-structure) + diff --git a/debug/accuracy_tools/msprobe/docs/03.config_examples.md b/debug/accuracy_tools/msprobe/docs/03.config_examples.md index 542250fac243f3ab2f1d0aff87bc509ac7c1a675..0d29a4eb1a824bba2c1bda1a214c9add2e87bdba 100644 --- a/debug/accuracy_tools/msprobe/docs/03.config_examples.md +++ b/debug/accuracy_tools/msprobe/docs/03.config_examples.md @@ -17,6 +17,7 @@ "statistics": { "scope": [], "list": [], + "tensor_list": [], "data_mode": ["all"], "summary_mode": "statistics" } diff --git a/debug/accuracy_tools/msprobe/docs/04.kernel_dump_PyTorch.md b/debug/accuracy_tools/msprobe/docs/04.kernel_dump_PyTorch.md index ce3fd54f5a6741b262f6248f70a9f1166ca0b4a6..346481aad12c42994669b7b3ea794843e49c1618 100644 --- a/debug/accuracy_tools/msprobe/docs/04.kernel_dump_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/04.kernel_dump_PyTorch.md @@ -6,7 +6,7 @@ ## 1 kernel dump 配置示例 -使用 kernel dump 时，list 必须要填一个 API 名称，kernel dump 目前每个 step 只支持采集一个 API 的数据。 +使用 kernel dump 时，task 需要配置为 tensor , list 必须要填一个 API 名称，kernel dump 目前每个 step 只支持采集一个 API 的数据。 API 名称填写参考 L1 dump 结果文件 dump.json 中的API名称，命名格式为：`{api_type}.{api_name}.{API调用次数}.{forward/backward}`。 ```json diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md index be9386df1d906330f76b536f19ef1a13d1553754..f5849d8a2bef03828d2a39dacce4aba2359b04aa 100644 --- a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md @@ -2,7 +2,7 @@ msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 -dump的'tensor'模式采集数据量大小，可以参考[数据量基线](./26.data_dump_PyTorch_baseline.md)。 +dump "statistics"模式的性能膨胀大小"与"tensor"模式采集的数据量大小，可以参考[dump基线](./26.data_dump_PyTorch_baseline.md)。 本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/pytorch/hook_module/support_wrap_ops.yaml 文件内手动修改，如下示例： @@ -15,6 +15,52 @@ functional: # functional为算子类别，找到对应的类别，在该类别 删除API的场景：部分模型代码逻辑会存在API原生类型校验，工具执行dump操作时，对模型的API封装可能与模型的原生API类型不一致，此时可能引发校验失败，详见《[FAQ](FAQ.md)》中“异常情况”的第10和11条。 +## 快速上手 + +这个示例定义了一个 nn.Module 类型的简单网络，使用原型函数 PrecisionDebugger 进行数据采集。 + +```python +# 根据需要import包 +import torch +import torch.nn as nn +import torch.nn.functional as F + +# 导入工具的数据采集接口 +from msprobe.pytorch import PrecisionDebugger, seed_all + +# 在模型训练开始前固定随机性 +seed_all() + +# 在模型训练开始前实例化PrecisionDebugger +debugger = PrecisionDebugger() + +# 定义网络 +class ModuleOP(nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear_1 = nn.Linear(in_features=8, out_features=4) + self.linear_2 = nn.Linear(in_features=4, out_features=2) + + def forward(self, x): + x1 = self.linear_1(x) + x2 = self.linear_2(x1) + r1 = F.relu(x2) + return r1 + +if __name__ == "__main__": + module = ModuleOP() + + # 开启数据 dump + debugger.start(model=module) + x = torch.randn(10, 8) + out = module(x) + loss = out.sum() + loss.backward() + + # 关闭数据 dump + debugger.stop() +``` + ## 1 接口介绍 ### 1.1 PrecisionDebugger @@ -30,9 +76,11 @@ PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, model 1. config_path：指定 dump 配置文件路径； 2. model：指定需要采集 Module 级数据的模型，支持传入 torch.nn.Module 或 list[torch.nn.Module] 类型，默认未配置。 level 配置为"L0"或"mix"时，必须在该接口或 **start** 接口中配置该参数。该参数在将来会从该接口移除，建议在 **start** 接口中配置该参数。 -3. 其他参数均在 [config.json](../config.json) 文件中可配，详细配置可见 [config.json 介绍](./02.config_introduction.md)。 +3. 其他参数均在 config.json 文件中可配，详细配置可见 [config.json 介绍](./02.config_introduction.md)。 + +此接口的参数均不是必要（均不配置的情况下默认采集所有 rank 和 step 的 L1 级别的统计数据），且优先级高于 config.json 文件中的配置，但可配置的参数相比 config.json 较少。 -此接口的参数均不是必要，且优先级高于 [config.json](../config.json) 文件中的配置，但可配置的参数相比 config.json 较少。 +注：此接口的初始化需与采集目标在同一个进程中，否则将无法采集目标数据。 ### 1.2 start @@ -41,12 +89,15 @@ level 配置为"L0"或"mix"时，必须在该接口或 **start** 接口中配置 **原型**： ```Python -debugger.start(model=None) +debugger.start(model=None, token_range=None) ``` 1. model：指定需要采集 Module 级数据的模型，支持传入 torch.nn.Module、list[torch.nn.Module]或Tuple[torch.nn.Module] 类型，默认未配置。 -level 配置为"L0"或"mix"时，必须在该接口或 **PrecisionDebugger** 接口中配置该参数。 +level 配置为"L0"|"mix"或token_range不为None时，必须在该接口或 **PrecisionDebugger** 接口中配置该参数。 本接口中的 model 比 PrecisionDebugger 中 model 参数优先级更高，会覆盖 PrecisionDebugger 中的 model 参数。 +
对于复杂模型，如果仅需要监控一部分(如model.A，model.A extends torch.nn.Module)，传入需要监控的部分(如model.A)即可。 +注意：传入的当前层不会被dump，工具只会dump传入层的子层级。如传入了model.A，A本身不会被dump，而是会dump A.x, A.x.xx等。 +2. token_range：指定推理模型采集时的token循环始末范围，支持传入[int, int]类型，代表[start, end]，范围包含边界，默认未配置。 ### 1.3 stop @@ -189,7 +240,7 @@ save(variable, name, save_backward=True) ### 1.10 set_init_step -**功能说明**：设置起始step数，step数默认从0开始计数，使用该接口后step从指定值开始计数。该函数需在 **start** 函数调用前使用，建议写在训练迭代的循环开始前。 +**功能说明**：设置起始step数，step数默认从0开始计数，使用该接口后step从指定值开始计数。该函数需要写在训练迭代的循环开始前，不能写在循环内。 **原型**： @@ -201,55 +252,47 @@ debugger.set_init_step(step) 1.step: 指定的起始step数。 +### 1.11 register_custom_api -## 2 示例代码 +**功能说明**：注册用户自定义的api到工具用于 L1 dump 。 -### 2.1 快速上手 +**原型**： -这个示例定义了一个 nn.Module 类型的简单网络，在进行数据采集时使用原型函数 PrecisionDebugger 传入 config_path 参数和 model 参数。 +```Python +debugger.register_custom_api(module, api_name, api_prefix) +``` +**参数说明**: -```python -# 根据需要import包 -import torch -import torch.nn as nn -import torch.nn.functional as F +以 torch.matmul api 为例 -# 导入工具的数据采集接口 -from msprobe.pytorch import PrecisionDebugger, seed_all +1.module: api 所属的包，即传入 torch。 -# 在模型训练开始前固定随机性 -seed_all() -# 在模型训练开始前实例化PrecisionDebugger -debugger = PrecisionDebugger(config_path='./config.json') +2.api_name: api 名，string类型，即传入 "matmul"。 -# 定义网络 -class ModuleOP(nn.Module): - def __init__(self) -> None: - super().__init__() - self.linear_1 = nn.Linear(in_features=8, out_features=4) - self.linear_2 = nn.Linear(in_features=4, out_features=2) +3.api_prefix: [dump.json](./27.dump_json_instruction.md) 中 api 名的前缀，可选，默认为包名的字符串格式, 即 "torch"。 - def forward(self, x): - x1 = self.linear_1(x) - x2 = self.linear_2(x1) - r1 = F.relu(x2) - return r1 +### 1.12 restore_custom_api -if __name__ == "__main__": - module = ModuleOP() - # 开启数据 dump - debugger.start(model=module) +**功能说明**：恢复用户原有的自定义的api，取消 dump 。 - x = torch.randn(10, 8) - out = module(x) - loss = out.sum() - loss.backward() +**原型**： - # 关闭数据 dump - debugger.stop() +```Python +debugger.restore_custom_api(module, api_name) ``` +**参数说明**: + +以 torch.matmul api 为例 + +1.module: api 所属的包，即传入 torch。 + +2.api_name: api 名，string类型，即传入 "matmul"。 + + +## 2 示例代码 -### 2.2 采集完整的前反向数据 + +### 2.1 采集完整的前反向数据 ```Python from msprobe.pytorch import PrecisionDebugger, seed_all @@ -270,7 +313,7 @@ for data, label in data_loader: debugger.step() # 结束一个step的dump ``` -### 2.3 采集指定代码块的前反向数据 +### 2.2 采集指定代码块的前反向数据 ```Python from msprobe.pytorch import PrecisionDebugger, seed_all @@ -294,7 +337,7 @@ for data, label in data_loader: debugger.step() # 结束一个step的dump ``` -### 2.4 采集函数模块化数据 +### 2.3 采集函数模块化数据 ```Python # 根据需要import包 @@ -336,6 +379,80 @@ if __name__ == "__main__": debugger.stop() ``` +### 2.4 跨文件采集数据 +为了确保所有API都被工具封装，PrecisionDebugger的实例化通常放在训练工程的入口位置，但有的时候，模型定义会在另一个文件中。 假设有两个文件，train.py（为训练工程入口）module.py（为模型定义文件），为了采集module.py中定义的ModuleOP模块中某些子模块或API的前反向数据，需要在train.py和module.py文件中分别导入PrecisionDebugger并进行如下配置。 + +train.py文件： + +```Python +# 根据需要import包 +import torch +from module import ModuleOP + +# 导入工具的数据采集接口 +from msprobe.pytorch import PrecisionDebugger + +# 将PrecisionDebugger的实例化放在文件的开始位置，即导包后的位置，确保所有API都被封装 +debugger = PrecisionDebugger(config_path='./config.json') + +if __name__ == "__main__": + module = ModuleOP() + + x = torch.randn(10, 8) + out = module(x) + loss = out.sum() + loss.backward() +``` + +module.py文件： + +```Python +import torch +import torch.nn as nn +import torch.nn.functional as F + +from msprobe.pytorch import PrecisionDebugger + +# 定义网络 +class ModuleOP(nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear_1 = nn.Linear(in_features=8, out_features=4) + self.linear_2 = nn.Linear(in_features=4, out_features=2) + + def forward(self, x): + PrecisionDebugger.start() + x1 = self.linear_1(x) + PrecisionDebugger.stop() + x2 = self.linear_2(x1) + r1 = F.relu(x2) + return r1 + +``` + +### 2.5 推理模型采集指定token_range + +```Python +from vllm import LLM, SamplingParams +from msprobe.pytorch import PrecisionDebugger, seed_all +# 在模型训练开始前固定随机性 +seed_all() +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") +# 模型定义及初始化等操作 +prompts = ["Hello, my name is"] +sampling_params = SamplingParams(temprature=0.8, top_p=0.95) +llm = LLM(model='...') +model = llm.llm_engine.model_executor.driver_worker.worker.model_runner.get_model() +# 开启数据dump, 指定采集推理模型逐字符循环推理中的第1~3次 +debugger.start(model=model, token_range=[1,3]) +# 推理模型生成的逻辑 +output = llm.generate(prompts, sampling_params=sampling_params) +# 关闭数据dump并落盘 +debugger.stop() +debugger.step() +``` + ## 3 dump 结果文件介绍 训练结束后，工具将 dump 的数据保存在 dump_path 参数指定的目录下。目录结构示例如下： @@ -349,8 +466,8 @@ if __name__ == "__main__": | | | | ├── Functional.linear.5.backward.output.pt # 命名格式为{api_type}.{api_name}.{API调用次数}.{forward/backward}.{input/output}.{参数序号}, 其中，“参数序号”表示该API的第n个输入或输出，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个元素。 | | | | ... | | | | ├── Module.conv1.Conv2d.forward.0.input.0.pt # 命名格式为{Module}.{module_name}.{class_name}.{forward/backward}.{调用次数}.{input/output}.{参数序号}, 其中，“参数序号”表示该Module的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该Module的第1个参数的第1个元素。 -| | | | ├── Module.conv1.Conv2D.forward.0.parameters.bias.pt # 模块参数数据：命名格式为{Module}.{module_name}.{class_name}.forward.{调用次数}.parameters.{parameter_name}。 -| | | | └── Module.conv1.Conv2D.parameters_grad.weight.pt # 模块参数梯度数据：命名格式为{Module}.{module_name}.{class_name}.parameters_grad.{parameter_name}。因为同一模块的参数使用同一梯度进行更新，所以参数梯度文件名不包含调用次数。 +| | | | ├── Module.conv1.Conv2d.forward.0.parameters.bias.pt # 模块参数数据：命名格式为{Module}.{module_name}.{class_name}.forward.{调用次数}.parameters.{parameter_name}。 +| | | | └── Module.conv1.Conv2d.parameters_grad.weight.pt # 模块参数梯度数据：命名格式为{Module}.{module_name}.{class_name}.parameters_grad.{parameter_name}。因为同一模块的参数使用同一梯度进行更新，所以参数梯度文件名不包含调用次数。 | | | | # 当dump时传入的model参数为List[torch.nn.Module]或Tuple[torch.nn.Module]时，模块级数据的命名中包含该模块在列表中的索引index，命名格式为{Module}.{index}.*，*表示以上三种模块级数据的命名格式，例如：Module.0.conv1.Conv2d.forward.0.input.0.pt。 │ | | ├── dump.json │ | | ├── stack.json diff --git a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md index f8670c93c308b76bb2f177a3342d1a85f8e868fb..ef420148ccf5310b22ef71db22faf17ce541acb5 100644 --- a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md @@ -26,18 +26,20 @@ msprobe 工具通过在训练脚本中添加 `PrecisionDebugger` 接口并启动训练的方式，采集模型在运行过程中的精度数据。该工具支持对MindSpore的静态图和动态图场景进行不同Level等级的精度数据采集。 -dump 的"tensor"模式采集数据量大小，可以参考[数据量基线](data_dump_MindSpore/data_dump_MindSpore_baseline.md)。 +dump "statistics"模式的性能膨胀大小"与"tensor"模式采集的数据量大小，可以参考[dump基线](data_dump_MindSpore/data_dump_MindSpore_baseline.md)。 ## 5. 场景介绍 ### 5.1 静态图场景 -在静态图场景下，msprobe 仅支持 **L2 Level** 的数据采集，且当 MindSpore 版本高于 2.5.0 时，必须使用编包时添加了`--include-mod=adump`选项的 mindstudio-probe whl 包进行 msprobe 工具安装。 +在静态图场景下，msprobe 支持 **L0 Level** 和 **L2 Level** 的数据采集。且当 MindSpore 版本高于 2.5.0 时，若需采集 **L2 Level** 数据，必须使用编包时添加了`--include-mod=adump`选项的 mindstudio-probe whl 包进行 msprobe 工具安装。 +- **L0 Level（Cell 级）** ：采集 `Cell` 对象的数据，适用于需要分析特定网络模块的情况。 + - **L2 Level（Kernel 级）** ：采集底层算子的输入输出数据，适用于深入分析算子级别的精度问题。 采集方式请参见[示例代码 > 静态图场景](#71-静态图场景)。详细介绍请参见[《config.json 配置文件介绍》](./02.config_introduction.md#11-通用配置)中的“level 参数”和[《config.json 配置示例》](./03.config_examples.md#2-mindspore-静态图场景) 中的“MindSpore 静态图场景”。 ### 5.2 动态图场景 -在动态图场景下，msprobe 支持 **L0** 、**L1** 、**mix** 、**L2 Level**、 **debug** 的数据采集，具体分为以下几种情况： +在动态图场景下，msprobe 支持 **L0** 、**L1** 、**mix** 、**L2**、 **debug** 的数据采集，具体分为以下几种情况： - **使用高阶 API（如 `Model 高阶API`）** ： - 需要使用 `MsprobeStep` 回调类来控制数据采集的启停，适用于 **L0** 、**L1** 、**mix** 、**L2** 数据采集。 @@ -56,7 +58,7 @@ dump 的"tensor"模式采集数据量大小，可以参考[数据量基线](data - **debug level （单点保存）**：单点保存网络中变量的正反向数据，适用于用户熟悉网络结构的场景。 -详细介绍请参见[《config.json 配置文件介绍》](./02.config_introduction.md#11-通用配置)中的“level 参数”和[《config.json 配置示例》](./03.config_examples.md#3-mindspore-动态图场景) 中的“MindSpore 动态图场景”。 +详细介绍请参见[《config.json 配置文件介绍》](./02.config_introduction.md#11-通用配置)中的“level 参数”。 ## 6 接口介绍 @@ -85,12 +87,15 @@ PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, step= **原型**： ```Python -start(model=None) +start(model=None, token_range=None) ``` **参数说明**: -1. model：指定需要采集数据的实例化模型，支持传入mindspore.nn.Cell、List[mindspore.nn.Cell]或Tuple[mindspore.nn.Cell] 类型, 默认未配置。Cell级别（"L0" level）dump 与 "mix" level dump 时，必须传入 model 才可以采集 model 内的所有Cell 对象数据。API级别（"L1" level）dump 时，传入 model 可以采集 model 内包含 primitive op 对象在内的所有 API 数据，若不传入 model 参数，则只采集非 primitive op 的 API 数据。 +1. model：指定需要采集数据的实例化模型，支持传入mindspore.nn.Cell、List[mindspore.nn.Cell]或Tuple[mindspore.nn.Cell] 类型，默认未配置。Cell级别（"L0" level）dump 与 "mix" level dump 时，必须传入 model 才可以采集 model 内的所有Cell 对象数据。API级别（"L1" level）dump 时，传入 model 可以采集 model 内包含 primitive op 对象在内的所有 API 数据，若不传入 model 参数，则只采集非 primitive op 的 API 数据。token_range不为None时，必须传入model参数。 +
对于复杂模型，如果仅需要监控一部分(如model.A，model.A extends mindspore.nn.Cell)，传入需要监控的部分(如model.A)即可。 +注意：传入的当前层不会被dump，工具只会dump传入层的子层级。如传入了model.A，A本身不会被dump，而是会dump A.x, A.x.xx等。 +2. token_range：指定推理模型采集时的token循环始末范围，支持传入[int, int]类型，代表[start, end]，范围包含边界，默认未配置。 #### 6.1.2 stop @@ -110,7 +115,7 @@ stop() **功能说明**：结束一个 step 的数据采集，完成所有数据落盘并更新 dump 参数。在一个 step 结束的位置添加，且必须在 **stop** 函数之后的位置调用。 该函数需要配合 **start** 和 **stop** 函数使用，尽量添加在反向计算代码之后，否则可能会导致反向数据丢失。 -**仅未使用 Model 高阶 API 的动态图场景支持。** +**仅未使用 Model 高阶 API 的动态图和静态图场景支持。** **原型**： @@ -151,7 +156,7 @@ save(variable, name, save_backward=True) #### 6.1.6 set_init_step -**功能说明**：设置起始step数，step数默认从0开始计数，使用该接口后step从指定值开始计数。该函数需在 **start** 函数调用前使用，建议写在训练迭代的循环开始前。 +**功能说明**：设置起始step数，step数默认从0开始计数，使用该接口后step从指定值开始计数。该函数需要写在训练迭代的循环开始前，不能写在循环内。 **原型**： @@ -164,9 +169,46 @@ set_init_step(step) 1.step: 指定的起始step数。 +#### 6.1.7 register_custom_api + +**功能说明**：注册用户自定义的api到工具，用于 L1 dump 。 + +**原型**： + +```Python +debugger.register_custom_api(module, api_name, api_prefix) +``` +**参数说明**: + +以 torch.matmul api 为例 + +1.module: api 所属的包，即传入 torch。 + +2.api_name: api 名，string类型，即传入 "matmul"。 + +3.api_prefix: [dump.json](./27.dump_json_instruction.md) 中 api 名的前缀，可选，默认为包名的字符串格式， 即 "torch"。 + +#### 6.1.8 restore_custom_api + +**功能说明**：恢复用户原有的自定义的api，取消 dump 。 + +**原型**： + +```Python +debugger.restore_custom_api(module, api_name) +``` +**参数说明**: + +以 torch.matmul api 为例 + +1.module: api 所属的包，即传入 torch。 + +2.api_name: api 名，string类型，即传入 "matmul"。 + + ### 6.2 msprobe.mindspore.MsprobeStep -**功能说明**：MindSpore Callback类，自动在每个step开始时调用start()接口，在每个step结束时调用stop()、step()接口。实现使用 Model 高阶 API 的动态图场景下 L0、L1、mix 级别的精度数据采集控制，控制粒度为单个 **Step** ，而 PrecisionDebugger.start, PrecisionDebugger.stop 接口的控制粒度任意训练代码段。 +**功能说明**：MindSpore Callback类，自动在每个step开始时调用start()接口，在每个step结束时调用stop()、step()接口。实现使用 Model 高阶 API 的动态图场景下 L0、L1、mix 级别，和静态图场景下 L0级别的精度数据采集控制，控制粒度为单个 **Step** ，而 PrecisionDebugger.start, PrecisionDebugger.stop 接口的控制粒度为任意训练代码段。 **原型**： @@ -209,6 +251,56 @@ seed_all(seed=1234, mode=False, rm_dropout=True) ### 7.1 静态图场景 +#### 7.1.1 L0 级别 + +**说明**: 静态图 L0 级别的Dump功能是基于mindspore.ops.TensorDump算子实现。在Ascend平台上的Graph模式下，可以通过设置环境变量 [MS_DUMP_SLICE_SIZE 和 MS_DUMP_WAIT_TIME](https://www.mindspore.cn/docs/zh-CN/r2.5.0/api_python/env_var_list.html) 解决在输出大Tesnor或输出Tensor比较密集场景下算子执行失败的问题。 + +##### 7.1.1.1 未使用 Model 高阶 API + + +```python +import mindspore as ms +ms.set_context(mode=ms.GRAPH_MODE, device_target="Ascend") + +from msprobe.mindspore import PrecisionDebugger +debugger = PrecisionDebugger(config_path="./config.json") + +# 模型、损失函数的定义以及初始化等操作 +# ... +model = Network() +# 数据集迭代的地方往往是模型开始训练的地方 +for data, label in data_loader: + debugger.start(model) # 进行 L0 级别下Cell 对象的数据采集时调用 + # 如下是模型每个 step 执行的逻辑 + grad_net = ms.grad(model)(data) + # ... + debugger.step() # 更新迭代数 +``` + +##### 7.1.1.2 使用 Model 高阶 API + + +```python +import mindspore as ms +from mindspore.train import Model +ms.set_context(mode=ms.GRAPH_MODE, device_target="Ascend") + +from msprobe.mindspore import PrecisionDebugger +from msprobe.mindspore.common.utils import MsprobeStep +debugger = PrecisionDebugger(config_path="./config.json") + +# 模型、损失函数的定义以及初始化等操作 +# ... + +model = Network() +# 进行 L0 级别下 Cell 对象的数据采集时调用 +debugger.start(model) +trainer = Model(model, loss_fn=loss_fn, optimizer=optimizer, metrics={'accuracy'}) +trainer.train(1, train_dataset, callbacks=[MsprobeStep(debugger)]) +``` + +#### 7.1.2 L2 级别 + ```python import mindspore as ms ms.set_context(mode=ms.GRAPH_MODE, device_target="Ascend") @@ -219,7 +311,8 @@ debugger.start() # 请勿将以上初始化流程置于模型实例化或 mindspore.communication.init 调用后 # 模型定义和训练代码 # ... - +debugger.stop() +debugger.step() ``` ### 7.2 动态图场景 @@ -318,11 +411,38 @@ trainer = Model(model, loss_fn=loss_fn, optimizer=optimizer, metrics={'accuracy' trainer.train(1, train_dataset) ``` + +#### 7.2.3 推理模型采集指定token_range +需要配合mindtorch套件改造原推理代码，套件包装后使用方式与torch一致，唯一区别为import的是msprobe.mindspore下的PrecisionDebugger。 + +```Python +from vllm import LLM, SamplingParams +from msprobe.mindspore import PrecisionDebugger, seed_all +# 在模型训练开始前固定随机性 +seed_all() +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") +# 模型定义及初始化等操作 +prompts = ["Hello, my name is"] +sampling_params = SamplingParams(temprature=0.8, top_p=0.95) +llm = LLM(model='...') +model = llm.llm_engine.model_executor.driver_worker.worker.model_runner.get_model() +# 开启数据dump, 指定采集推理模型逐字符循环推理中的第1~3次 +debugger.start(model=model, token_range=[1,3]) +# 推理模型生成的逻辑 +output = llm.generate(prompts, sampling_params=sampling_params) +# 关闭数据dump并落盘 +debugger.stop() +debugger.step() +``` + ## 8. dump 结果文件介绍 ### 8.1 静态图场景 -训练结束后，数据将保存在 `dump_path` 指定的目录下。 +训练结束后，数据将保存在 `dump_path` 指定的目录下。
+L0 级别 dump 的目录结构与动态图场景下目录结构一致。
+L2 级别 dump 的目录结构如下所示： 若jit_level=O2，MindSpore 版本不低于 2.5.0，且使用mindstudio-probe发布包或源码编包时添加了`--include-mod=adump`选项，目录结构示例如下： ``` @@ -369,9 +489,9 @@ dump 结果目录结构示例如下： | | | | ├── Tensor.__add__.0.forward.output.0.npy | | | | ... | | | | ├── Jit.AlexNet.0.forward.input.0.npy -| | | | ├── Primitive.conv2d.Conv2D.0.forward.input.0.npy -| | | | ├── Cell.conv1.Conv2D.forward.0.parameters.weight.npy # 模块参数数据：命名格式为{Cell}.{cell_name}.{class_name}.forward.{调用次数}.parameters.{parameter_name}。 -| | | | ├── Cell.conv1.Conv2D.parameters_grad.weight.npy # 模块参数梯度数据：命名格式为{Cell}.{cell_name}.{class_name}.parameters_grad.{parameter_name}。因为同一模块的参数使用同一梯度进行更新，所以参数梯度文件名不包含调用次数。 +| | | | ├── Primitive.conv2d.Conv2d.0.forward.input.0.npy +| | | | ├── Cell.conv1.Conv2d.forward.0.parameters.weight.npy # 模块参数数据：命名格式为{Cell}.{cell_name}.{class_name}.forward.{调用次数}.parameters.{parameter_name}。 +| | | | ├── Cell.conv1.Conv2d.parameters_grad.weight.npy # 模块参数梯度数据：命名格式为{Cell}.{cell_name}.{class_name}.parameters_grad.{parameter_name}。因为同一模块的参数使用同一梯度进行更新，所以参数梯度文件名不包含调用次数。 | | | | └── Cell.relu.ReLU.forward.0.input.0.npy # 命名格式为{Cell}.{cell_name}.{class_name}.{forward/backward}.{调用次数}.{input/output}.{参数序号}, 其中，“参数序号”表示该Cell的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该Cell的第1个参数的第1个元素。 | | | | # 当dump时传入的model参数为List[mindspore.nn.Cell]或Tuple[mindspore.nn.Cell]时，模块级数据的命名中包含该模块在列表中的索引index，命名格式为{Cell}.{index}.*，*表示以上三种模块级数据的命名格式，例如：Cell.0.relu.ReLU.forward.0.input.0.npy。 │ | | ├── dump.json @@ -393,13 +513,13 @@ dump 结果目录结构示例如下： * `rank`：设备 ID，每张卡的数据保存在对应的 `rank{ID}` 目录下。非分布式场景下没有 rank ID，目录名称为 rank。 * `dump_tensor_data`：保存采集到的张量数据。 -* `dump.json`： 保存API或Cell前反向数据的统计量信息。包含dump数据的API名称或Cell名称，各数据的dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置summary_mode="md5"时的CRC-32数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#2-MindSpore场景下的dump.json文件)。 +* `dump.json`： 保存API或Cell前反向数据的统计量信息。包含dump数据的API名称或Cell名称，各数据的dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置summary_mode="md5"时的CRC-32数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#2-mindspore-场景下的-dumpjson-文件)。 * `stack.json`：API/Cell的调用栈信息。 * `construct.json`：分层分级结构，level为L1时，construct.json内容为空。 dump 过程中，npy 文件在对应API或者模块被执行后就会落盘，而 json 文件则需要在正常执行 PrecisionDebugger.stop() 后才会写入完整数据，因此，程序异常终止时，被执行API对应的 npy 文件已被保存，但 json 文件中的数据可能丢失。 -动态图场景下使能 PSJit 或 PIJit，装饰特定 Cell 或 function，被装饰的部分会全部/部分使能**静态图**流程。 +动态图场景下使能 PSJit 或 PIJit（MindSpore 自 2.6 版本起将其更名为 Ast 或 Bytecode，但为了保持术语一致，下文仍沿用原有称谓），装饰特定 Cell 或 function，被装饰的部分会全部/部分使能**静态图**流程。 - PSJit 场景下 config.json 文件配置 level 为 L1 时，被 PSJit 装饰的部分也作为 API 被 dump 到对应目录；配置 level 为 L2 时，则只会 dump 用户网络中静态图流程下的相关 kernel，其结果目录同jit_level 为 O0/O1 时的静态图 dump 相同。 - PIJit 场景下 config.json 文件配置 level 为 L1 时，会被还原为动态图，按 API 粒度进行 dump；配置 level 为 L2 时，则只会 dump 用户网络中静态图流程下的相关 kernel。 @@ -414,6 +534,7 @@ npy文件名的前缀含义如下： | Primitive | mindspore.ops.Primitive API数据 | | Mint | mindspore.mint API数据 | | MintFunctional | mindspore.mint.nn.functional API数据 | +| MintDistributed | mindspore.mint.distributed API数据 | | Distributed | mindspore.communication.comm_func API数据 | | Jit | 被"jit"装饰的模块或函数数据 | | Cell | mindspore.nn.Cell 类（模块）数据 | diff --git a/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_PyTorch.md b/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_PyTorch.md index a93ad3b62405d549a16e7196e2f2145de68e8674..d5ee3cacc02daef6267aa7d79469d7d2f7b5b5a8 100644 --- a/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_PyTorch.md @@ -3,7 +3,7 @@ ## 1 简介 为了应对大模型场景下，通过离线预检方式 dump API 输入输出数据导致的存储资源紧张问题，提供在线精度预检功能。本功能实现在执行 NPU 训练操作的过程中，通过 TCP/IP 协议在 NPU -Host 与 GPU Host 设备间建立连接，将 NPU 上对应 API 的输入数据在 GPU 设备上运行，将两份输出数据进行比对，得到预检比对结果，从而减少数据 dump 的步骤，降低存储资源的占用。针对偏差较大的算子，两方比对（NPU vs. GPU）的方法缺少裁判进行裁定。 参考离线预检，在线预检场景同时支持两方比对和三方比对方式，按照 api 的精度标准要求，选择比对两方比对和三方比对。 +Host 与 GPU Host 设备间建立连接，将 NPU 上对应 API 的输入数据在 GPU 设备上运行，将两份输出数据进行比对，得到预检比对结果，从而减少数据 dump 的步骤，降低存储资源的占用。针对偏差较大的算子，两方比对（NPU vs. GPU）的方法缺少裁判进行裁定。 参考离线预检，在线预检场景同时支持两方比对和三方比对方式，按照 api 的精度标准要求，选择两方比对或三方比对。 ## 2 在线精度预检流程 @@ -60,10 +60,12 @@ Host 与 GPU Host 设备间建立连接，将 NPU 上对应 API 的输入数据 #### 3.1.3 局域网场景配置示例 -若采用 TLS1.2 协议加密传输 api 数据，需配置 SSL 证书，可参考如下生成自签名证书方法，仅供调试使用，生产环境请申请正式证书。 +若采用 TLS1.2 协议加密传输 api 数据，需配置 SSL 证书，可参考如下生成自签名证书方法。 + +以下秘钥生成方法仅为简单示例，客户应使用与自己需求相符的秘钥生成和存储机制并保证秘钥安全性与机密性，必要时可采用分层秘钥机制。 ```shell # 创建私钥文件server.key -openssl genrsa -out server.key 2048 +openssl genrsa -out server.key 3072 # 创建签名请求文件server.csr openssl req -new -key server.key -out server.csr diff --git a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md index 3bf65032edae2b8e35c5818d5c030c9ce4c79e95..d2f938459410a3a1cc4c363975b9b10939d9e7fe 100644 --- a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md @@ -34,9 +34,18 @@ msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result | -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。对其中的mint api以及部分Tensor api进行预检，预检支持的Tensor api列表详见 [ 预检支持列表](../mindspore/api_accuracy_checker/checker_support_api.yaml)。 | str | 是 | | -o 或 --out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | | -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | str | 否 | +| -save_error_data | 保存(随机数据模式)精度未达标的 API 输入输出数据。 | 空 | 否 | 预检执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API Name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见 [4 预检结果](#4-预检结果)。 +随机数据模式下，如果需要保存比对不达标的输入和输出数据，可以在 run_ut 执行命令结尾添加 `-save_error_data`，例如： + +```bash +msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result -save_error_data +``` + +数据默认会存盘到 '{out_path}/error_data' 路径下。 + ### 3.2 使用 multi_run_ut 执行多线程预检 multi_run_ut 脚本，可以并行在多个Device执行 run_ut 操作，从而减少预检耗时。示例如下： @@ -45,16 +54,19 @@ multi_run_ut 脚本，可以并行在多个Device执行 run_ut 操作，从而 msprobe -f mindspore multi_run_ut -api_info ./dump.json -d 0 1 2 3 ``` -| 参数名称 | 说明 |参数类型 | 是否必选 | -| ---------------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------- | ---------------------------------- | -| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。对其中的mint api以及部分Tensor api进行预检，预检支持的Tensor api列表详见 [ 预检支持列表](../mindspore/api_accuracy_checker/checker_support_api.yaml)。 | str | 是 | -| -o 或 --out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | -| -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | str | 否 | -| -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0，支持同时指定 0 ~ Device数量 - 1 ，例如 0 1 2 3 4。 | List[int] | 否 | +| 参数名称 | 说明 | 参数类型 | 是否必选 | +| ---------------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------| ---------------------------------- | +| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。对其中的mint api以及部分Tensor api进行预检，预检支持的Tensor api列表详见 [ 预检支持列表](../mindspore/api_accuracy_checker/checker_support_api.yaml)。 | str | 是 | +| -o 或 --out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | +| -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | str | 否 | +| -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0，支持同时指定 0 ~ Device数量 - 1 ，例如 0 1 2 3 4。 | List[int] | 否 | +| -save_error_data | 保存(随机数据模式)精度未达标的 API 输入输出数据。 | 空 | 否 | 在不同卡数下，使用38B语言大模型的预检耗时基线参考 [multi_run_ut耗时基线](accuracy_checker_MindSpore/accuracy_checker_MindSpore_baseline.md) +数据默认会存盘到 './ut_error_data{timestamp}' 路径下 + ### 3.3 断点续检 断点续检操作通过如下命令执行： diff --git a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md index 6f886215b0a389582bc3cc4c31943f76e6a414a3..fcb76842196b2d96b54f17c54580de459be31df4 100644 --- a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md @@ -51,14 +51,14 @@ msprobe -f pytorch compare -i ./compare.json -o ./output -s 完整参数说明： -| 参数名 | 说明 | 是否必选 | -|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | -| -i 或 --input_path | 指定[比对文件](#214-比对文件)，str 类型。 | 是 | -| -o 或 --output_path | 配置比对结果文件存盘目录，str 类型，默认在当前目录创建output目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 否 | -| -s 或 --stack_mode | 比对结果展示调用栈信息（NPU_Stack_Info）的开关，bool 类型。单卡场景开启时，根据[比对文件](#214-比对文件)的参数说明配置stack_path；多卡场景开启时，自动识别npu_dump目录下stack.json文件，如存在生成详细调用栈信息，否则不生成，此参数不生效。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | +| 参数名 | 说明 | 是否必选 | +|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | +| -i 或 --input_path | 指定[比对文件](#51-比对文件)，str 类型。 | 是 | +| -o 或 --output_path | 配置比对结果文件存盘目录，str 类型，默认在当前目录创建output目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。
提示：output目录下与结果件同名文件将被删除覆盖。 | 否 | +| -s 或 --stack_mode | 比对结果展示调用栈信息（NPU_Stack_Info）的开关，bool 类型。单卡场景开启时，根据[比对文件](#51-比对文件)的参数说明配置stack_path；多卡场景开启时，自动识别npu_dump目录下stack.json文件，如存在生成详细调用栈信息，否则不生成，此参数不生效。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | | -c 或 --compare_only | 仅比对开关，bool 类型。该参数默认未配置，会启用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度可能不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。通过配置该参数取消自动精度分析，仅输出比对结果表格。 | 否 | -| -f 或 --fuzzy_match | 模糊匹配，bool 类型。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | -| -dm或--data_mapping | 自定义映射关系比对。需要指定自定义映射文件*.yaml。自定义映射文件的格式请参见[自定义映射文件](#215-自定义映射文件)。仅[API和模块无法自动匹配场景](#213-api和模块无法自动匹配场景)需要配置。仅支持逐卡比对，即使用[比对文件](#214-比对文件)的单卡场景示例。 | 否 | +| -f 或 --fuzzy_match | 模糊匹配，bool 类型。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | +| -dm或--data_mapping | 自定义映射关系比对。需要指定自定义映射文件*.yaml。自定义映射文件的格式请参见[自定义映射文件](#52-自定义映射文件)。仅[API和模块无法自动匹配场景](#213-api和模块无法自动匹配场景)需要配置。仅支持逐卡比对，即使用[比对文件](#51-比对文件)的单卡场景示例。 | 否 | #### 2.1.2 整网比对场景 @@ -70,7 +70,7 @@ msprobe -f pytorch compare -i ./compare.json -o ./output -s 2. 参见 [PyTorch 场景下的数据采集](./05.data_dump_PyTorch.md)章节完成 CPU 或 GPU 与 NPU 的精度数据 dump。 -3. 创建[比对文件](#214-比对文件)。 +3. 创建[比对文件](#51-比对文件)。 4. 运行命令： @@ -88,7 +88,7 @@ msprobe -f pytorch compare -i ./compare.json -o ./output -s 2. 参见[PyTorch 场景下的数据采集](./05.data_dump_PyTorch.md)章节完成 CPU 或 GPU 与 NPU 的精度数据 dump。 -3. 创建[比对文件](#214-比对文件)（单卡场景示例）。 +3. 创建[比对文件](#51-比对文件)（单卡场景示例）。 4. 运行命令： @@ -96,75 +96,32 @@ msprobe -f pytorch compare -i ./compare.json -o ./output -s msprobe -f pytorch compare -i ./compare.json -o ./output -s -dm data_mapping.yaml ``` - data_mapping.yaml文件配置请参见[自定义映射文件](#215-自定义映射文件)。 + data_mapping.yaml文件配置请参见[自定义映射文件](#52-自定义映射文件)。 该场景不支持-f模糊匹配。 5. 查看比对结果，请参见 [3 精度比对结果分析](#3-精度比对结果分析)。 -#### 2.1.4 比对文件 - - 以在当前目录创建 ./compare.json 为例。 - - - 单卡场景示例： - - ```json - { - "npu_path": "./npu_dump/dump.json", - "bench_path": "./bench_dump/dump.json", - "stack_path": "./npu_dump/stack.json", - "is_print_compare_log": true - } - ``` - - - 多卡场景示例： - - ```json - { - "npu_path": "./npu_dump/step0", - "bench_path": "./bench_dump/step0", - "is_print_compare_log": true - } - ``` -**参数说明**： +#### 2.1.4 单点数据比对场景 -| 参数名 | 说明 | 是否必选 | -| -------------------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------| -| npu_path | 配置 NPU 环境下的 dump.json 文件（单卡场景）或真实数据目录（多卡场景），str 类型。 | 是 | -| bench_path | 配置 CPU、GPU 或 NPU 环境下的 dump.json 文件（单卡场景）或真实数据目录（多卡场景），str 类型。 | 是 | -| stack_path | 配置 NPU dump 目录下的 stack.json 文件，str 类型。如果没有配置stack_path，命令行-s参数不生效，程序自动识别是否存在stack.json文件，如存在，则比对结果中呈现NPU_Stack_Info，如不存在，则不呈现。如果配置了stack_path，比对结果中是否呈现NPU_Stack_Info则通过命令行参数-s来控制。 | 否 | -| is_print_compare_log | 配置是否开启单个算子的日志打屏。可取值 true 或 false，默认为 true。关闭后则只输出常规日志，bool 类型。 | 否 | +单点数据比对场景是指：CPU 或 GPU 与 NPU环境的网络中单点保存的数据比对。 -#### 2.1.5 自定义映射文件 +支持单卡和多卡，可同时比对多卡的单点数据。多机场景需要每个设备单独执行比对操作。 -文件名格式：*.yaml，*为文件名，可自定义。 - -文件内容格式： - -```yaml -# API -{api_type}.{api_name}.{API调用次数}.{前向反向}.{input/output}.{参数序号}: {api_type}.{api_name}.{API调用次数}.{前向反向}.{input/output}.{参数序号} -# 模块 -{Module}.{module_name}.{前向反向}.{index}.{input/output}.{参数序号}: {Module}.{module_name}.{前向反向}.{index}.{input/output}.{参数序号} -``` +1. 配置[config.json](../config.json)文件。 -冒号左侧和右侧分别为PyTorch框架不同版本或不同芯片环境的API的名称和module模块名称。 +2. 参见 [单点保存工具](./28.debugger_save_instruction.md)章节完成 CPU 或 GPU 与 NPU 的单点数据采集。 -API和模块名称请从《[PyTorch 场景的精度数据采集](05.data_dump_PyTorch.md)》中的dump.json文件获取。 +3. 创建[比对文件（单点数据）](#53-比对文件单点数据)。 -文件内容示例： - -```yaml -# API -NPU.npu_fusion_attention.4.forward.input.0: NPU.npu_fusion_attention.4.forward.input.0 -# 模块 -Module.module.language_model.embedding.word_embedding.VocabParallelEmbedding.forward.0.input.0: Module.module.language_model.embedding.word_embedding.VocabParallelEmbedding.forward.0.input.0 -``` +4. 运行命令： -API和模块名称在dump.json文件中的“data_name”字段展示，如下图红框处所示： + ```shell + msprobe -f pytorch compare -i ./compare.json -o ./output + ``` -![pt_dump](./img/pt_dump.png) +5. 查看比对结果，请参见 [3 精度比对结果分析](#3-精度比对结果分析)。 ### 2.2 比对函数方式 @@ -180,13 +137,13 @@ compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_mat **参数说明**： -| 参数名 | 说明 | 是否必选 | -| ------------ |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | +| 参数名 | 说明 | 是否必选 | +| ------------ |-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | | input_param | 配置 dump 数据文件及目录，dict 类型。配置参数包括：
"npu_json_path"：指定 NPU dump 目录下的 dump.json 文件。
**配置示例**："npu_json_path": "./npu_dump/dump.json"。
"bench_json_path"：指定 CPU、GPU 或 NPU dump 目录下的 dump.json 文件。
**配置示例**："bench_json_path": "./bench_dump/dump.json"。
"stack_json_path"：指定 NPU dump 目录下的 stack.json 文件。
**配置示例**："stack_json_path": "./npu_dump/stack.json"。
"is_print_compare_log"：配置是否开启单个算子的日志打屏。
**配置示例**：True 或 False。 | 是 | -| output_path | 配置比对结果文件存盘目录，str 类型。
**配置示例**：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | -| stack_mode | 配置 stack_mode 的开关，bool 类型。仅当配置 stack_json_path 时需要，开启时比对结果呈现NPU_Stack_Info，关闭时不呈现。当不配置stack_json_path 时，自动识别是否存在stack.json，存在时呈现NPU_Stack_Info，否则不呈现。
**配置示例**：stack_mode=True，默认为 False。 | 否 | -| auto_analyze | 自动精度分析，bool 类型。开启后工具自动针对比对结果进行分析，识别到第一个精度可能不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 advisor_{timestamp}.txt 文件）。
**配置示例**：auto_analyze=False，默认为 True。 | 否 | -| fuzzy_match | 模糊匹配，bool 类型。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。
**配置示例**：fuzzy_match=True，默认为 False。 | 否 | +| output_path | 配置比对结果文件存盘目录，str 类型。
**配置示例**：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。
提示：output目录下与结果件同名文件将被删除覆盖。 | 是 | +| stack_mode | 配置 stack_mode 的开关，bool 类型。仅当配置 stack_json_path 时需要，开启时比对结果呈现NPU_Stack_Info，关闭时不呈现。当不配置stack_json_path 时，自动识别是否存在stack.json，存在时呈现NPU_Stack_Info，否则不呈现。
**配置示例**：stack_mode=True，默认为 False。 | 否 | +| auto_analyze | 自动精度分析，bool 类型。开启后工具自动针对比对结果进行分析，识别到第一个精度可能不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 advisor_{timestamp}.txt 文件）。
**配置示例**：auto_analyze=False，默认为 True。 | 否 | +| fuzzy_match | 模糊匹配，bool 类型。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。
**配置示例**：fuzzy_match=True，默认为 False。 | 否 | **函数示例**： @@ -215,12 +172,12 @@ compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs) **参数说明**： -| 参数名 | 说明 | 是否必选 | -| -------------- |-----------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | -| npu_dump_dir | 配置 NPU 环境下的 dump 目录。str 类型。dump 数据目录须指定到 step 级。
**配置示例**：'./npu_dump/step0'。 | 是 | -| bench_dump_dir | 配置 CPU、GPU 或 NPU 环境下的 dump 目录。str 类型。
**配置示例**：'./gpu_dump/step0'。 | 是 | -| output_path | 配置比对结果文件存盘目录。需要预先创建 output_path 目录。str 类型。
**配置示例**：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_rank{npu_ID}-rank{cpu/gpu/npu_ID}_{timestamp}.xlsx`。 | 是 | -| **kwargs | 支持 compare 的所有可选参数。 其中，stack_mode不生效，自动识别是否存在stack.json，如存在，呈现NPU_Stack_Info，否则不呈现。 | 否 | +| 参数名 | 说明 | 是否必选 | +| -------------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | +| npu_dump_dir | 配置 NPU 环境下的 dump 目录。str 类型。dump 数据目录须指定到 step 级。
**配置示例**：'./npu_dump/step0'。 | 是 | +| bench_dump_dir | 配置 CPU、GPU 或 NPU 环境下的 dump 目录。str 类型。
**配置示例**：'./gpu_dump/step0'。 | 是 | +| output_path | 配置比对结果文件存盘目录。需要预先创建 output_path 目录。str 类型。
**配置示例**：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_rank{npu_ID}_{timestamp}.xlsx`。
提示：output目录下与结果件同名文件将被删除覆盖。 | 是 | +| **kwargs | 支持 compare 的所有可选参数。 其中，stack_mode不生效，自动识别是否存在stack.json，如存在，呈现NPU_Stack_Info，否则不呈现。 | 否 | **函数示例**： @@ -336,7 +293,7 @@ MD5 模式： 4. MaxRelativeErr：当最大相对误差越接近 0 表示其计算的误差越小。 - 当 dump 数据中存在 0 或 Nan 时，比对结果中最大相对误差则出现 inf 或 Nan 的情况，属于正常现象。 + 当 dump 数据中存在 0 或 nan 时，比对结果中最大相对误差则出现 inf 或 nan 的情况，属于正常现象。 5. One Thousandth Err Ratio（相对误差小于千分之一的元素比例）、Five Thousandths Err Ratio（相对误差小于千分之五的元素比例）精度指标：是指 NPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一、千分之五的比例占总元素个数的比例。该数据仅作为精度下降趋势的参考，并不参与计算精度是否通过的判定。 @@ -360,11 +317,11 @@ msprobe -f pytorch merge_result -i ./input_dir -o ./output_dir -config ./config. **完整参数说明** -| 参数名 | 说明 | 是否必选 | -| ---------------------- |------------------------------------------------------------------------------------| -------- | -| -i 或 --input_dir | 多卡比对结果存盘目录，即使用compare比对的结果输出目录，str类型。所有比对结果应全部为真实数据比对结果或统计数据比对结果，否则可能导致汇总数据不完整。 | 是 | -| -o 或 --output_dir | 数据提取汇总结果存盘目录，str类型。文件名称基于时间戳自动生成，格式为：`multi_ranks_compare_merge_{timestamp}.xlsx`。 | 是 | -| -config或--config-path | 指定需要汇总数据的API和比对指标的yaml文件路径，str类型。
yaml文件详细介绍见下文“**yaml文件说明**”。 | 是 | +| 参数名 | 说明 | 是否必选 | +| ---------------------- |-------------------------------------------------------------------------------------------------------------------| -------- | +| -i 或 --input_dir | 多卡比对结果存盘目录，即使用compare比对的结果输出目录，str类型。所有比对结果应全部为真实数据比对结果或统计数据比对结果，否则可能导致汇总数据不完整。 | 是 | +| -o 或 --output_dir | 数据提取汇总结果存盘目录，str类型。文件名称基于时间戳自动生成，格式为：`multi_ranks_compare_merge_{timestamp}.xlsx`。
提示：output目录下与结果件同名文件将被删除覆盖。 | 是 | +| -config或--config-path | 指定需要汇总数据的API和比对指标的yaml文件路径，str类型。
yaml文件详细介绍见下文“**yaml文件说明**”。 | 是 | **yaml文件说明** @@ -380,10 +337,10 @@ compare_index: - MeanRelativeErr ``` -| 参数名 | 说明 | -| ------------- | ------------------------------------------------------------ | -| api | 表示需要汇总的API或module名称。如果没有配置，工具会提示报错。
api名称配置格式为：`{api_type}.{api_name}.{API调用次数}.{前向反向}`
须按顺序配置以上四个字段，可按如下组合配置：
{api_type}
{api_type}.{api_name}
{api_type}.{api_name}.{API调用次数}
{api_type}.{api_name}.{API调用次数}.{前向反向}
这里的api指代API或module。 | -| compare_index | 表示需要汇总的比对指标。compare_index需为dump_mode对应比对指标的子集。如果没有配置，工具将根据比对结果自动提取dump_mode对应的全部比对指标进行汇总。
统计数据模式比对指标：Max diff、Min diff、Mean diff、Norm diff、MaxRelativeErr、MinRelativeErr、MeanRelativeErr、NormRelativeErr
真实数据模式比对指标：Cosine、MaxAbsErr、MaxRelativeErr、One Thousandth Err Ratio、Five Thousandths Err Ratio | +| 参数名 | 说明 | +| ------------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| api | 表示需要汇总的API或module名称。如果没有配置，工具会提示报错。
api名称配置格式为：`{api_type}.{api_name}.{API调用次数}.{前向反向}`
须按顺序配置以上四个字段，可按如下组合配置：
{api_type}
{api_type}.{api_name}
{api_type}.{api_name}.{API调用次数}
{api_type}.{api_name}.{API调用次数}.{前向反向}
这里的api指代API或module。 | +| compare_index | 表示需要汇总的比对指标。compare_index需为dump_mode对应比对指标的子集。如果没有配置，工具将根据比对结果自动提取dump_mode对应的全部比对指标进行汇总。
统计数据模式比对指标：Max diff、Min diff、Mean diff、L2norm diff、MaxRelativeErr、MinRelativeErr、MeanRelativeErr、NormRelativeErr
真实数据模式比对指标：Cosine、EucDist、MaxAbsErr、MaxRelativeErr、One Thousandth Err Ratio、Five Thousandths Err Ratio | **汇总结果件说明** @@ -414,4 +371,180 @@ compare_index: 6. Distributed.broadcast：输入为要广播的数据，输出为广播后的数据。 7. Distributed.isend：点对点通信，输入为要发送的数据，输出为发送的数据。 8. Distributed.irecv：点对点通信，输入为原数据，输出为接收的新数据。 -9. Distributed.all_to_all_single：输出数据为所有卡上的数据切分后合并的结果。 \ No newline at end of file +9. Distributed.all_to_all_single：输出数据为所有卡上的数据切分后合并的结果。 + +## 5 附录 + +### 5.1 比对文件 + + 以在当前目录创建 ./compare.json 为例。 + + - 单卡场景示例： + + ```json + { + "npu_path": "./npu_dump/dump.json", + "bench_path": "./bench_dump/dump.json", + "stack_path": "./npu_dump/stack.json", + "is_print_compare_log": true + } + ``` + + - 多卡场景示例： + + ```json + { + "npu_path": "./npu_dump/step0", + "bench_path": "./bench_dump/step0", + "is_print_compare_log": true + } + ``` + +### 5.2 自定义映射文件 + +文件名格式：*.yaml，*为文件名，可自定义。 + +文件内容格式： + +```yaml +# API +{api_type}.{api_name}.{API调用次数}.{前向反向}.{input/output}.{参数序号}: {api_type}.{api_name}.{API调用次数}.{前向反向}.{input/output}.{参数序号} +# 模块 +{Module}.{module_name}.{前向反向}.{index}.{input/output}.{参数序号}: {Module}.{module_name}.{前向反向}.{index}.{input/output}.{参数序号} +``` + +冒号左侧和右侧分别为PyTorch框架不同版本或不同芯片环境的API的名称和module模块名称。 + +API和模块名称请从《[PyTorch 场景的精度数据采集](05.data_dump_PyTorch.md)》中的dump.json文件获取。 + +文件内容示例： + +```yaml +# API +NPU.npu_fusion_attention.4.forward.input.0: NPU.npu_fusion_attention.4.forward.input.0 +# 模块 +Module.module.language_model.embedding.word_embedding.VocabParallelEmbedding.forward.0.input.0: Module.module.language_model.embedding.word_embedding.VocabParallelEmbedding.forward.0.input.0 +``` + +当dump.json文件中存在“data_name”字段时，API和模块名称为data_name字段去掉文件后缀，如下图红框处所示： + +![pt_dump](./img/pt_dump.png) + +当dump.json文件中不存在“data_name”字段时，名称的拼写规则如下： + +input_args、input_kwargs和output使用统一的命名规则，当值是list类型时，名称后面添加'.{index}'，当值类型是dict类型时，名称后面加'.{key}'，当值类型是具体Tensor或null或int或float或bool或空list/dict等时，命名结束。 + +以下面api的dump文件为例： +```yaml + "Functional.max_pool2d.0.forward": { + "input_args": [ + { + "type": "torch.Tensor", + "dytpe": "torch_float32", + "shape": [ + 1, + 64, + 14, + 14 + ], + "Max": xxx, + "Min": xxx, + "Mean": xxx, + "Norm": xxx, + "requires_grad": true + }, + { + "type": "int", + "value": 3 + }, + { + "type": "int", + "value": 2 + }, + { + "type": "int", + "value": 1 + }, + { + "type": "int", + "value": 1 + } + ], + "input_kwargs": { + "ceil_mode": { + "type": "bool", + "value": false + }, + "return_indices": { + "type": "bool", + "value": false + }, + }, + "output": [ + { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 1, + 64, + 7, + 7 + ], + "Max": xxx, + "Min": xxx, + "Mean": xxx, + "Norm": xxx, + "requires_grad": true + } + ] + } +``` + +初始名称为Functional.max_pool2d.0.forward，input_args是list，长度为5，第0项后面是Tensor，命名结束；第1-4项后面均是int，命名结束；按照顺序命名为 +``` +Functional.max_pool2d.0.forward.input.0 +Functional.max_pool2d.0.forward.input.1 +Functional.max_pool2d.0.forward.input.2 +Functional.max_pool2d.0.forward.input.3 +Functional.max_pool2d.0.forward.input.4 +``` +input_kwargs是dict，key是ceil_mode、return_indices，值均是bool，命名结束；命名为 +``` +Functional.max_pool2d.0.forward.input.ceil_mode +Functional.max_pool2d.0.forward.input.return_indices +``` +output是list，长度为1，第0项后面是Tensor，命名结束；按照顺序命名为 +``` +Functional.max_pool2d.0.forward.output.0 +``` +综上，生成的的op_name为 +``` +Functional.max_pool2d.0.forward.input.0 +Functional.max_pool2d.0.forward.input.1 +Functional.max_pool2d.0.forward.input.2 +Functional.max_pool2d.0.forward.input.3 +Functional.max_pool2d.0.forward.input.4 +Functional.max_pool2d.0.forward.input.ceil_mode +Functional.max_pool2d.0.forward.input.return_indices +Functional.max_pool2d.0.forward.output.0 +``` + +### 5.3 比对文件（单点数据） + + - 单卡场景示例： + + ```json + { + "npu_path": "./npu_dump/debug.json", + "bench_path": "./bench_dump/debug.json" + } + ``` + + - 多卡场景示例(step0目录下包含debug.json文件)： + + ```json + { + "npu_path": "./npu_dump/step0", + "bench_path": "./bench_dump/step0" + } + ``` \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md b/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md index 1b1824a774f15a86106585669d5f3412b3faca2e..7eb96de60593db235c5bc5fac489c56c318b5955 100644 --- a/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md @@ -19,7 +19,7 @@ msprobe精度比对工具主要用于如下场景： - 通过对同一个网络模型，在整网环境下分别在MindSpore动态图和PyTorch环境下获得API或模块dump数据，由用户指定可以比对的API或模块，以PyTorch数据作为标杆，进行自动比对，从而实现跨框架的精度对比。 - 通过对同一个网络模型，在整网环境下分别在MindSpore动态图和PyTorch环境下获得API或模块dump数据，由用户指定可以比对的模型代码中的Layer层，以PyTorch数据作为标杆，进行自动比对，从而实现跨框架的精度对比。 -执行精度比对操作需要安装msprobe工具。详见《[MindStudio精度调试工具](../README.md)》的“工具安装”章节。 +执行精度比对操作需要安装msprobe工具。详见[《msprobe 工具安装指南》](./01.installation.md)。 ## 2 命令行比对 @@ -35,17 +35,17 @@ msprobe -f mindspore compare -i ./compare.json -o ./output -s **完整参数说明** -| 参数名 | 说明 | 是否必选 | -| -------------------- | ------------------------------------------------------------ | -------- | -| -i或--input_path | 指定比对文件。比对文件内容及示例请参见[比对文件](#31-比对文件)或[比对文件（kernel）](#32-比对文件kernel)（比对文件（kernel）仅[不同版本下的全量kernel比对](#23-不同版本下的全量kernel比对)场景支持）。 | 是 | -| -o或--output_path | 配置比对结果文件存盘目录，默认会在当前目录创建output目录。文件名称基于时间戳自动生成，格式为：
`compare_result_{timestamp}.xlsx`
`compare_result_{rank_id}_{step_id}_{timestamp}.xlsx`（仅[不同版本下的全量kernel比对](#23-不同版本下的全量kernel比对)场景支持）。 | 否 | -| -s或--stack_mode | 比对结果展示调用栈信息（NPU_Stack_Info）的开关，bool 类型。单卡场景开启时，需要使用[比对文件](#31-比对文件)的单卡场景配置stack_path指定stack.json文件，才能生成详细调用栈信息，否则在比对时会报错；暂不支持多卡场景。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | -| -c或--compare_only | 仅比对开关，bool 类型。该参数默认未配置，会启用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度可能不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。通过配置该参数取消自动精度分析，仅输出比对结果表格。 | 否 | -| -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | -| -am或--api_mapping | 跨框架比对。配置该参数时表示开启跨框架API比对功能，可以指定自定义映射文件*.yaml，不指定映射文件时按照msprobe定义的默认映射关系进行比对。自定义映射文件的格式请参见[自定义映射文件（api_mapping）](#33-自定义映射文件api_mapping)。仅[跨框架的API比对](#25-跨框架的api比对)场景需要配置。 | 否 | -| -cm或--cell_mapping | 跨框架比对。配置该参数时表示开启跨框架cell模块比对功能，可以指定自定义映射文件*.yaml，不指定映射文件时按照msprobe定义的默认映射关系进行比对。自定义映射文件的格式请参见[自定义映射文件（cell_mapping）](#34-自定义映射文件cell_mapping)。仅[跨框架的cell模块比对](#26-跨框架的cell模块比对)场景需要配置。 | 否 | -| -dm或--data_mapping | 同框架或跨框架比对。通过映射文件指定两个具体参数的对应关系，可以在L0、L1或mix采集场景下使用。配置该参数的同时需要指定自定义映射文件*.yaml。自定义映射文件的格式请参见[自定义映射文件（data_mapping）](#35-自定义映射文件data_mapping)。 | 否 | -| -lm或--layer_mapping | 跨框架比对。配置该参数时表示开启跨框架Layer层的比对功能，指定模型代码中的Layer层后，可以识别对应dump数据中的模块或API。需要指定自定义映射文件*.yaml。自定义映射文件的格式请参见[自定义映射文件（Layer_mapping）](#36-自定义映射文件layer_mapping)。仅[跨框架的Layer层比对](#27-跨框架的layer层比对)场景需要配置。 | 否 | +| 参数名 | 说明 | 是否必选 | +| -------------------- |--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | +| -i或--input_path | 指定比对文件。比对文件内容及示例请参见[比对文件](#41-比对文件)或[比对文件（kernel）](#42-比对文件kernel)（比对文件（kernel）仅[不同版本下的全量kernel比对](#23-不同版本下的全量kernel比对)场景支持）。 | 是 | +| -o或--output_path | 配置比对结果文件存盘目录，默认会在当前目录创建output目录。文件名称基于时间戳自动生成，格式为：
`compare_result_{timestamp}.xlsx`
`compare_result_{rank_id}_{step_id}_{timestamp}.xlsx`（仅[不同版本下的全量kernel比对](#23-不同版本下的全量kernel比对)场景支持）。
提示：output目录下与结果件同名文件将被删除覆盖。 | 否 | +| -s或--stack_mode | 比对结果展示调用栈信息（NPU_Stack_Info）的开关，bool 类型。单卡场景开启时，需要使用[比对文件](#41-比对文件)的单卡场景配置stack_path指定stack.json文件，才能生成详细调用栈信息，否则在比对时会报错；暂不支持多卡场景。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | +| -c或--compare_only | 仅比对开关，bool 类型。该参数默认未配置，会启用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度可能不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。通过配置该参数取消自动精度分析，仅输出比对结果表格。 | 否 | +| -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | +| -am或--api_mapping | 跨框架比对。配置该参数时表示开启跨框架API比对功能，可以指定自定义映射文件*.yaml，不指定映射文件时按照msprobe定义的默认映射关系进行比对。自定义映射文件的格式请参见[自定义映射文件（api_mapping）](#43-自定义映射文件api_mapping)。仅[跨框架的API比对](#25-跨框架的api比对)场景需要配置。 | 否 | +| -cm或--cell_mapping | 跨框架比对。配置该参数时表示开启跨框架cell模块比对功能，可以指定自定义映射文件*.yaml，不指定映射文件时按照msprobe定义的默认映射关系进行比对。自定义映射文件的格式请参见[自定义映射文件（cell_mapping）](#44-自定义映射文件cell_mapping)。仅[跨框架的cell模块比对](#26-跨框架的cell模块比对)场景需要配置。 | 否 | +| -dm或--data_mapping | 同框架或跨框架比对。通过映射文件指定两个具体参数的对应关系，可以在L0、L1或mix采集场景下使用。配置该参数的同时需要指定自定义映射文件*.yaml。自定义映射文件的格式请参见[自定义映射文件（data_mapping）](#45-自定义映射文件data_mapping)。 | 否 | +| -lm或--layer_mapping | 跨框架比对。配置该参数时表示开启跨框架Layer层的比对功能，指定模型代码中的Layer层后，可以识别对应dump数据中的模块或API。需要指定自定义映射文件*.yaml。自定义映射文件的格式请参见[自定义映射文件（Layer_mapping）](#46-自定义映射文件layer_mapping)。仅[跨框架的Layer层比对](#27-跨框架的layer层比对)场景需要配置。 | 否 | 动态图模式没有填写任何mapping时，按照同框架比对的方式进行比对，比对数据和标杆数据的Cell或Api名称需要完全相同才能匹配得上。 @@ -53,7 +53,7 @@ msprobe -f mindspore compare -i ./compare.json -o ./output -s 1. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》完成不同环境下MindSpore静态图精度数据的采集，得到不同框架版本的API dump数据。 -2. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 +2. 创建比对文件，文件内容及示例请参见[比对文件](#41-比对文件)。 3. 执行如下示例命令进行比对： @@ -67,7 +67,7 @@ msprobe -f mindspore compare -i ./compare.json -o ./output -s 1. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》完成不同环境下MindSpore静态图精度数据的采集，得到不同框架版本的kernel dump数据。 -2. 创建比对文件，文件内容及示例请参见[比对文件（kernel）](#32-比对文件kernel)。 +2. 创建比对文件，文件内容及示例请参见[比对文件（kernel）](#42-比对文件kernel)。 3. 执行如下示例命令进行比对： @@ -85,7 +85,7 @@ msprobe -f mindspore compare -i ./compare.json -o ./output -s 2. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》完成不同环境下MindSpore动态图精度数据的采集，得到不同框架版本的cell模块dump数据。 -3. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 +3. 创建比对文件，文件内容及示例请参见[比对文件](#41-比对文件)。 4. 执行如下示例命令进行比对： @@ -101,7 +101,7 @@ msprobe -f mindspore compare -i ./compare.json -o ./output -s 2. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》和《[PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)》完成不同环境下API精度数据的采集，得到两个框架的API dump数据。 -3. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 +3. 创建比对文件，文件内容及示例请参见[比对文件](#41-比对文件)。 4. 执行如下示例命令进行比对： @@ -115,14 +115,14 @@ msprobe -f mindspore compare -i ./compare.json -o ./output -s msprobe -f mindspore compare -i ./compare.json -o ./output -s -am api_mapping.yaml ``` - api_mapping.yaml文件配置请参见[自定义映射文件（api_mapping）](#33-自定义映射文件api_mapping)。 + api_mapping.yaml文件配置请参见[自定义映射文件（api_mapping）](#43-自定义映射文件api_mapping)。 不传入api_mapping.yaml的情况下将按照内置的api映射进行匹配；传入api_mapping.yaml的情况下优先按照api_mapping.yaml的内容进行匹配，api_mapping.yaml中没有涉及的按照内置的api映射进行匹配。 此外，也可以通过data_mapping.yaml文件实现具体参数的匹配，例： ```shell msprobe -f mindspore compare -i ./compare.json -o ./output -s -dm data_mapping.yaml ``` - data_mapping.yaml的写法请参见[自定义映射文件（data_mapping）](#35-自定义映射文件data_mapping)。 + data_mapping.yaml的写法请参见[自定义映射文件（data_mapping）](#45-自定义映射文件data_mapping)。 5. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)》章节。 @@ -132,7 +132,7 @@ msprobe -f mindspore compare -i ./compare.json -o ./output -s 2. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》和《[PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)》完成不同环境下cell模块精度数据的采集，得到两个框架的cell模块dump数据。 -3. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 +3. 创建比对文件，文件内容及示例请参见[比对文件](#41-比对文件)。 4. 执行如下示例命令进行比对： @@ -146,14 +146,14 @@ msprobe -f mindspore compare -i ./compare.json -o ./output -s msprobe -f mindspore compare -i ./compare.json -o ./output -s -cm cell_mapping.yaml ``` - cell_mapping.yaml文件配置请参见[自定义映射文件（cell_mapping）](#34-自定义映射文件cell_mapping)。 + cell_mapping.yaml文件配置请参见[自定义映射文件（cell_mapping）](#44-自定义映射文件cell_mapping)。 不传入cell_mapping.yaml的情况下仅将Cell改成Module后进行匹配；传入cell_mapping.yaml的情况下将按照cell_mapping.yaml的内容进行匹配。 此外，也可以通过data_mapping.yaml文件实现具体参数的匹配，例： ```shell msprobe -f mindspore compare -i ./compare.json -o ./output -s -dm data_mapping.yaml ``` - data_mapping.yaml的写法请参见[自定义映射文件（data_mapping）](#35-自定义映射文件data_mapping)。 + data_mapping.yaml的写法请参见[自定义映射文件（data_mapping）](#45-自定义映射文件data_mapping)。 5. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)》章节。 @@ -165,7 +165,7 @@ layer_mapping可以从Layer层识别整网的API和Cell，简化配置。 2. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》和《[PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)》完成不同环境下API或模块精度数据的采集，得到两个框架的API或模块dump数据。 -3. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 +3. 创建比对文件，文件内容及示例请参见[比对文件](#41-比对文件)。 4. 执行如下示例命令进行比对： @@ -173,16 +173,29 @@ layer_mapping可以从Layer层识别整网的API和Cell，简化配置。 msprobe -f mindspore compare -i ./compare.json -o ./output -s -lm layer_mapping.yaml ``` - layer_mapping.yaml文件配置请参见[自定义映射文件（layer_mapping）](#36-自定义映射文件layer_mapping)。 + layer_mapping.yaml文件配置请参见[自定义映射文件（layer_mapping）](#46-自定义映射文件layer_mapping)。 此外，也可以通过data_mapping.yaml文件实现具体参数的匹配，例： ```shell msprobe -f mindspore compare -i ./compare.json -o ./output -s -dm data_mapping.yaml ``` - data_mapping.yaml的写法请参见[自定义映射文件（data_mapping）](#35-自定义映射文件data_mapping)。 + data_mapping.yaml的写法请参见[自定义映射文件（data_mapping）](#45-自定义映射文件data_mapping)。 5. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)》章节。 +### 2.8 单点数据比对 +1. 参见 [单点保存工具](./28.debugger_save_instruction.md)章节完成 CPU 或 GPU 与 NPU 的单点数据采集。 + +2. 创建比对文件，文件内容及示例请参见[比对文件(单点数据)](#47-比对文件单点数据)。 + +3. 执行如下示例命令进行比对： + + ```shell + msprobe -f mindspore compare -i ./compare.json -o ./output + ``` + +4. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)》章节。 + ## 3 多卡比对结果提取汇总通信算子数据 本功能是将多卡比对场景的比对结果，进行通信算子数据提取和汇总，输出整理好的通信算子多卡比对精度表。 @@ -204,11 +217,11 @@ msprobe -f mindspore merge_result -i ./input_dir -o ./output_dir -config ./confi **完整参数说明** -| 参数名 | 说明 | 是否必选 | -| ---------------------- | ------------------------------------------------------------ | -------- | -| -i 或 --input_dir | 多卡比对结果存盘目录，即使用compare比对的结果输出目录，str类型。所有比对结果应全部为真实数据比对结果或统计数据比对结果，否则可能导致汇总数据不完整。 | 是 | -| -o 或 --output_dir | 数据提取汇总结果存盘目录，str类型。文件名称基于时间戳自动生成，格式为：`multi_ranks_compare_merge_{timestamp}.xlsx`。 | 是 | -| -config或--config-path | 指定需要汇总数据的API和比对指标的yaml文件路径，str类型。
yaml文件详细介绍见下文“**yaml文件说明**”。 | 是 | +| 参数名 | 说明 | 是否必选 | +| ---------------------- |-------------------------------------------------------------------------------------------------------------------| -------- | +| -i 或 --input_dir | 多卡比对结果存盘目录，即使用compare比对的结果输出目录，str类型。所有比对结果应全部为真实数据比对结果或统计数据比对结果，否则可能导致汇总数据不完整。 | 是 | +| -o 或 --output_dir | 数据提取汇总结果存盘目录，str类型。文件名称基于时间戳自动生成，格式为：`multi_ranks_compare_merge_{timestamp}.xlsx`。
提示：output目录下与结果件同名文件将被删除覆盖。 | 是 | +| -config或--config-path | 指定需要汇总数据的API和比对指标的yaml文件路径，str类型。
yaml文件详细介绍见下文“**yaml文件说明**”。 | 是 | **yaml文件说明** @@ -224,10 +237,10 @@ compare_index: - MeanRelativeErr ``` -| 参数名 | 说明 | -| ------------- | ------------------------------------------------------------ | -| api | 表示需要汇总的API或module名称。如果没有配置，工具会提示报错。
api名称配置格式为：`{api_type}.{api_name}.{API调用次数}.{前向反向}`
须按顺序配置以上四个字段，可按如下组合配置：
{api_type}
{api_type}.{api_name}
{api_type}.{api_name}.{API调用次数}
{api_type}.{api_name}.{API调用次数}.{前向反向}
这里的api指代API或module。 | -| compare_index | 表示需要汇总的比对指标。compare_index需为dump_mode对应比对指标的子集。如果没有配置，工具将根据比对结果自动提取dump_mode对应的全部比对指标进行汇总。
统计数据模式比对指标：Max diff、Min diff、Mean diff、Norm diff、MaxRelativeErr、MinRelativeErr、MeanRelativeErr、NormRelativeErr
真实数据模式比对指标：Cosine、MaxAbsErr、MaxRelativeErr、One Thousandth Err Ratio、Five Thousandths Err Ratio | +| 参数名 | 说明 | +| ------------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| api | 表示需要汇总的API或module名称。如果没有配置，工具会提示报错。
api名称配置格式为：`{api_type}.{api_name}.{API调用次数}.{前向反向}`
须按顺序配置以上四个字段，可按如下组合配置：
{api_type}
{api_type}.{api_name}
{api_type}.{api_name}.{API调用次数}
{api_type}.{api_name}.{API调用次数}.{前向反向}
这里的api指代API或module。 | +| compare_index | 表示需要汇总的比对指标。compare_index需为dump_mode对应比对指标的子集。如果没有配置，工具将根据比对结果自动提取dump_mode对应的全部比对指标进行汇总。
统计数据模式比对指标：Max diff、Min diff、Mean diff、L2norm diff、MaxRelativeErr、MinRelativeErr、MeanRelativeErr、NormRelativeErr
真实数据模式比对指标：Cosine、EucDist、MaxAbsErr、MaxRelativeErr、One Thousandth Err Ratio、Five Thousandths Err Ratio | **汇总结果件说明** @@ -289,8 +302,8 @@ compare_index: | 参数名 | 说明 | 是否必选 | | -------------------- | ------------------------------------------------------------ |------| -| npu_path | 配置NPU环境下的dump.json文件（单卡场景）。跨框架场景指定为MindSpore的json文件。数据类型：str。 | 是 | -| bench_path | 配置CPU、GPU或NPU环境下的dump.json文件（单卡场景）。 跨框架场景指定为PyTorch的json文件。数据类型：str。 | 是 | +| npu_path | 配置NPU环境下的dump.json或debug.json文件（单卡场景）。跨框架场景指定为MindSpore的json文件。数据类型：str。 | 是 | +| bench_path | 配置CPU、GPU或NPU环境下的dump.json或debug.json文件（单卡场景）。 跨框架场景指定为PyTorch的json文件。数据类型：str。 | 是 | | stack_path | 配置NPU dump目录下的stack.json文件。数据类型：str。 如果没有配置stack_path，命令行-s参数不生效，程序自动识别是否存在stack.json文件，如存在，则比对结果中呈现NPU_Stack_Info，如不存在，则不呈现。如果配置了stack_path，比对结果中是否呈现NPU_Stack_Info则通过命令行参数-s来控制。 | 否 | | is_print_compare_log | 配置是否开启单个算子的日志打屏。可取值true或false，默认为true。关闭后则只输出常规日志。数据类型：bool | 否 | @@ -573,7 +586,7 @@ input_args、input_kwargs和output使用统一的命名规则，当值是list类 "md5": "28f8f74f" } ] -} +} ``` , 初始名称为`Cell.network.module.NetworkWithLoss.forward.0`，`input_args`是`list`，长度为2，按照顺序命名为 @@ -646,4 +659,22 @@ yaml文件中只需配置MindSpore与PyTorch模型代码中功能一致但名称 模型代码示例： -![ms_dump](./img/ms_layer.png) \ No newline at end of file +![ms_dump](./img/ms_layer.png) + +### 4.7 比对文件（单点数据） + +单卡场景示例如下： + ```json +{ +"npu_path": "./npu_dump/debug.json", +"bench_path": "./bench_dump/debug.json" +} + ``` + +多卡场景（step0目录下包含debug.json文件）示例如下： +```json +{ +"npu_path": "./npu_dump/step0", +"bench_path": "./bench_dump/step0" +} +``` \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md b/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md index 983477554e138f3e547f2d3efcf14fdfc4a991a0..a1cd8db8f0d7ade2d5f45161b68581c25996a423 100644 --- a/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md @@ -12,13 +12,13 @@ msprobe 工具在 PyTorch 场景下提供溢出数据采集功能和溢出数据 ### 1.2 接口介绍 -溢出检测功能提供的接口与数据采集任务一致，详见[ PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)中的"**1 接口介绍**"章节。 +溢出检测功能提供的接口与数据采集任务一致，详见[ PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)中的"**接口介绍**"章节。 其中 PrecisionDebugger 中的 task 或是 config.json 中的 task 需要指定为 **overflow_check**，详见[配置文件介绍](./02.config_introduction.md)中的 "**1.1 通用配置介绍**"和"**1.5 task 配置为 overflow_check**"章节。 ### 1.3 示例代码 -溢出检测功能使用方式与数据采集任务一致，详见[ PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)中的"**2 示例代码**"章节。 +溢出检测功能使用方式与数据采集任务一致，详见[ PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)中的"**示例代码**"章节。 ### 1.4 结果文件介绍 diff --git a/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md b/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md index 3b674a35e40e8e79b37a43ade7525219a45ee38e..ab280f1119cd17634a9a45aa48ad7e4ec78facb6 100644 --- a/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md @@ -18,13 +18,13 @@ export MS_ASCEND_CHECK_OVERFLOW_MODE="INFNAN_MODE" ## 1 接口介绍 -溢出检测功能提供的接口与数据采集任务一致，详见MindSpore 场景的精度数据采集中的["**1 接口介绍**"](./06.data_dump_MindSpore.md#1-接口介绍)章节。 +溢出检测功能提供的接口与数据采集任务一致，详见MindSpore 场景的精度数据采集中的["**接口介绍**"](./06.data_dump_MindSpore.md#6-接口介绍)章节。 需要注意，目前暂不支持动态图 "L1" level 下 primitive op 的溢出检测。 ## 2 示例代码 -溢出检测功能使用方式与数据采集任务一致，详见MindSpore 场景的精度数据采集中的["**2 示例代码**"](./06.data_dump_MindSpore.md#2-示例代码)节。 +溢出检测功能使用方式与数据采集任务一致，详见MindSpore 场景的精度数据采集中的["**示例代码**"](./06.data_dump_MindSpore.md#7-示例代码)节。 ## 3 溢出检测结果文件介绍 diff --git a/debug/accuracy_tools/msprobe/docs/14.data_parse_PyTorch.md b/debug/accuracy_tools/msprobe/docs/14.data_parse_PyTorch.md index 68a3d1a57dc1b649ffdb6d02d7be378900458e65..9e46fc309e73680f1cf10ef6a4ec355e39b2cdd4 100644 --- a/debug/accuracy_tools/msprobe/docs/14.data_parse_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/14.data_parse_PyTorch.md @@ -26,13 +26,7 @@ Parse >>> - 支持交互式指定 pkl 文件中 API 对应 dump 数据查看。 - 支持 API 进行可选层级比对和打印（统计级和像素级）。 -Ctrl+C 可以退出 parse 交互式界面。不退出 parse 交互式界面若需要执行非该界面下的内置 Shell 命令，且命令与 parse 交互式界面命令冲突时，非该界面命令需要使用 run 命令，在相关命令前加上 run 前缀，如下示例： - -```bash -msprobe -f pytorch parse -Parse >>> run vim cli.py -Parse >>> vim cli.py -``` +Ctrl+C 可以退出 parse 交互式界面。 ### 2.2 kernel 层级算子数据批量转换 diff --git a/debug/accuracy_tools/msprobe/docs/17.grad_probe.md b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md index f210088013415e40167f3eea3aab6163b0c947dc..da1183617610c61a41d6e0b27cf070fb9644112a 100644 --- a/debug/accuracy_tools/msprobe/docs/17.grad_probe.md +++ b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md @@ -65,6 +65,7 @@ + 值分布：梯度数据落在各个区间的元素个数占总元素个数的比例。 + bounds：一个列表，用来划分出区间以统计值分布。例如传入bounds = [-10, 0, 10]，此时有一个 grad_value: Tensor = [9.3 , 5.4, -1.0, -12.3]，依据 bounds 划分出 (-inf, -10]、(-10, 0]、(0, 10]、(10, inf) 四个区间，然后统计grad_value里的数据落在每个区间内的个数，得到 1、1、2、0。如下图所示： + ![Alt text](./img/grad_probe_image-1.png) 2. 插入代码。示例代码如下： diff --git a/debug/accuracy_tools/msprobe/docs/18.online_dispatch.md b/debug/accuracy_tools/msprobe/docs/18.online_dispatch.md index e686c61b68add9c9a1ade9ae3e89b897c9b8d6bf..4d1833a3fbd32c37aaf5dc7c0993c176417ab584 100644 --- a/debug/accuracy_tools/msprobe/docs/18.online_dispatch.md +++ b/debug/accuracy_tools/msprobe/docs/18.online_dispatch.md @@ -70,15 +70,15 @@ PyTorch NPU在线精度比对是msprobe工具实现在PyTorch训练过程中直 | api_list | dump范围，dump_mode="list"时设置，需要Dump Aten Ir API名称，默认为None，Aten Ir API名称可以通过dir(torch.ops.aten)查看。 | 否 | | dump_path| dump文件生成的路径。 | 是 | | tag | 传入tag字符串，成为dump文件夹名一部分，默认为None。 | 否 | -| process_num | 多进程并发数，默认为0。 | 否 | +| process_num | 多进程并发数，默认为0，最大不超过CPU核数的四分之一。 | 否 | | debug | debug信息打印，默认为False。 | 否 | ### dump数据存盘说明 -dump数据存盘目录名格式：`atat_tag_rankid_{timestamp}`。 +dump数据存盘目录名格式：`msprobe_rankid_{timestamp}`。 子目录下包含1个比对结果csv文件、cpu和npudump数据目录，npu目录下包含Aten IR在NPU上的输入输出的dump数据，由于CPU的输入是直接使用NPU的输入执行，因此cpu目录下只包含执行输出的dump数据。 ```bash -atat_rank4_20230911170521 +msprobe_rank4_20230911170521 ├── compare_result_rank4_20230911170521.csv ├── cpu │ ├── native_batch_norm_backward_10_output.0.npy diff --git a/debug/accuracy_tools/msprobe/docs/19.monitor.md b/debug/accuracy_tools/msprobe/docs/19.monitor.md index 099bba30c0dbbf7fce8b52891897b3ad48ae04a9..0febb8d9c2f837003eb358d10d1457af5722eca2 100644 --- a/debug/accuracy_tools/msprobe/docs/19.monitor.md +++ b/debug/accuracy_tools/msprobe/docs/19.monitor.md @@ -10,7 +10,7 @@ 要求： - PyTorch场景：torch不低于**2.0** -- MindSpore场景：mindspore不低于**2.4.10**，仅支持**MindSpore动态图**，暂不支持**msadapter**套件 +- MindSpore场景：mindspore不低于**2.4.10**，仅支持**MindSpore动态图**，已支持**msadapter**套件 ## 功能介绍 下表中字段为训练状态轻量化监控工具的完整功能点： @@ -21,12 +21,11 @@ | [权重梯度监控](#权重梯度监控) | 开启权重梯度监控 | PyTorch、MindSpore | | [激活值监控](#激活值监控) | 开启激活值监控 | PyTorch、MindSpore | | [优化器状态监控](#优化器状态监控) | 开启优化器状态监控 | PyTorch、MindSpore | +| [采集module堆栈信息](#采集module堆栈信息) | 采集监控的第一个 step 的 module 对应的堆栈信息辅助问题定位 | PyTorch、MindSpore | | [指定监控对象](#指定监控对象) | 指定监控的nn.Module(nn.Cell)及对应的输入输出 | PyTorch、MindSpore | | [打印模型结构](#打印模型结构) | 打印模型结构 | PyTorch | -| [Module全量监控](#Module全量监控) | 对全量module的输入输出做监控 | PyTorch、MindSpore | -| [Parameter全量监控](#Parameter全量监控) | 对全量Parameter的输入输出做监控 | PyTorch、MindSpore | -| [输出格式和统计量](#输出格式和统计量) | format PyTorch支持`csv`、`tensorboard`和`api`，MindSpore仅支持`csv`，`ops`均支持，`ndigits`仅PyTorch支持 | PyTorch、MindSpore | -| [梯度异常时序判断](#梯度异常时序判断) | 梯度异常时自动梯度落盘 | PyTorch | +| [输出格式和统计量](#输出格式和统计量) | format PyTorch支持`csv`、`tensorboard`和`api`，MindSpore仅支持`csv`，`ops`、`ndigits`均支持 | PyTorch、MindSpore | +| [异常告警](#异常告警) | 监控对象指标异常时自动告警，支持异常数据落盘 | PyTorch、MindSpore | | [csv格式数据转tensorboard可视化显示](#csv格式数据转tensorboard可视化显示) | 将csv转为tensorboard文件显示 | PyTorch | | [动态启停](#动态启停) | 训练过程中动态修改配置开启监控 | PyTorch、MindSpore | | [功能重载](#功能重载) | 训练中开启激活值监控。待废弃，请使用动态启停功能代替。 | PyTorch | @@ -205,12 +204,26 @@ monitor.monitor_gnorm_with_ad( 本工具针对分布式计算框架megatron和deepspeed框架做了适配，暂不支持其他框架。 +### 采集module堆栈信息 +- 工具配置示例： +```json +{ + "targets": { + }, + "format": "csv", + "stack_info": true +} +``` +开启 `stack_info` 后会采集监控的第一个 step 的所有 module 的堆栈信息，输出格式仅支持 csv 。 ## 高阶功能 + ### 指定监控对象 -工具支持对nn.Module（**激活值监控**）和nn.Parameter（**权重监控**、**权重梯度监控、优化器监控**）对象实现相应的监控行为，在配置文件的"targets"（dict）字段指定，targets格式为{module_name/param_name: {filed: format}}。 +工具支持对指定nn.Module进行状态监控，在配置文件的`targets`字段中指定，`targets`格式为{module_name: {}}。 + +module_name可以通过nn.Module的接口named_modules()获取。 #### 打印模型结构 工具提供可选项`print_struct`打印模型结构，帮助配置targets。工具会在在第一个step后打印结构并停止训练进程，模型结构默认打印在`$MONITOR_OUTPUT_DIR/module_struct.json`。 @@ -221,7 +234,6 @@ monitor.monitor_gnorm_with_ad( ``` 输出样例: -字段`config`用于配置文件中指定module target。其余为各个元素的shape和dtype。 ```json "0:63.mlp.linear_fc2": { @@ -245,40 +257,30 @@ monitor.monitor_gnorm_with_ad( } }, ``` +对于module对象，通常关心前向/反向传播的输入和输出： -- Module - 对于module对象，通常关心其前向的输入(input)输出(output)和反向的输入--前向输出的梯度(output_grad)和输出--前向输入的梯度（input_grad）。同时需要声明这些对象的类型，通常为"tensor"或"tuple\[length]"。 +- 前向的输入(input) +- 前向的输出(output) +- 反向的输入，表示前向输出的梯度(output_grad) +- 反向的输出，表示前向输入的梯度(input_grad) - "tensor"可以直接用来计算统计量，"tuple"需要进一步指定监控的索引。如"tuple[2]:0",表示该对象为长度2的tuple，对第0元素进行监控；不指定索引时，默认对第0元素进行监控。 - module_name可以通过nn.Module的接口`named_modules()`获取。 -```json -// 示例：对一个名为"module.encoder.layers.0.mlp"的module，监控其前向输入第0元素和输出。 -{ - "targets": { - "module.encoder.layers.0.mlp": { - "input": "tuple[2]:0", - "output": "tensor" - } - } -} -``` -#### Module全量监控 -工具提供简便的全量module监控方式。或不配置targets、all_xy字段，同样表示全量监控。 +#### 指定监控对象 + +targets字段指定监控对象示例如下： ```json -{ - "targets": {}, - "all_xy": true +// 示例：对一个名为"module.encoder.layers.0.mlp"的module。 +"targets": { + "module.encoder.layers.0.mlp": {} } ``` +对于parameter对象，通常会关注其在一个训练迭代中的梯度（weight grad）、adam类优化器中的动量（1st moment, 2nd moment）。 +parameter归属于某一module，可以通过指定module_name来监控包含在这一module中的**所有**parameter。 -- Parameter - 对于parameter对象，通常会关注其在一个训练迭代中的梯度（weight grad）、adam类优化器中的动量（1st moment, 2nd moment）。 - parameter归属于某一module，也可以通过指定module_name来监控包含在这一module中的**所有**parameter。 +param_name可以通过nn.Module的接口`named_parameters()`获取。 - param_name可以通过nn.Module的接口`named_parameters()`获取。 ```json // 示例：监控"module.encoder.layers.0.mlp"的所有参数和"module.embedding.word_embedding.weight"这一参数 { @@ -289,8 +291,9 @@ monitor.monitor_gnorm_with_ad( } ``` -#### Parameter全量监控 -工具提供简便的全量parameter监控方式。或不配置targets，同样表示全量监控。 +#### 全量监控 + +工具提供简便的全量module对象监控方式。 ```json { @@ -298,7 +301,9 @@ monitor.monitor_gnorm_with_ad( } ``` + ### 输出格式和统计量 + 工具配置示例： ```json { @@ -333,7 +338,7 @@ export MONITOR_OUTPUT_DIR=/xxx/output_dir 监控结果写入csv文件中，可以通过`ndigits`字段设置小数位数。 表头为 vpp_stage | name | step | micro_step(optional) | *ops |。 仅在激活值监控的输出文件中包含micor_step。 - 激活值监控的name为.\, 其他任务的name为> + 激活值监控的name为.\, 其他任务的name为 - **api** 监控结果不落盘，在训练过程中可以通过`generate_wgrad_metrics`、`generate_xy_metrics`等接口获取，使用方式参考[公开接口](#公开接口) 。 @@ -349,16 +354,36 @@ export MONITOR_OUTPUT_DIR=/xxx/output_dir ![step_count_per_record](img/monitor/step_count_per_record.png) -### 梯度异常时序判断 +### 异常告警 +工具的异常告警功能旨在自动判断训练过程中的异常现象，用户可通过在配置文件中配置alert字段来指定告警规则，并在训练过程中根据该规则及时打屏对用户发出告警。 + + 1. 训练前配置相关参数 -工具支持自动判断训练过程中的梯度异常，需要在配置文件中设置alert相关字段。"AnomalyTurbulence"会将当前数值与历史均值比较，如果相对偏差超过阈值，会在打屏信息中提示用户。如果打开"`dump`"选项，则会将异常梯度相关信息落盘到目录`monitor_output/anomaly_detected`，用于后续时序判断。 +当前支持的异常告警规则如下： + +| 异常告警 |解释| rule_name | args是否可选 | +|--------------|----|-----------|---------------------------------------------------------------------| +| 历史均值偏离告警 |将当前数值与历史均值比较。如果相对偏差超过阈值，会在打屏信息中提示用户| AnomalyTurbulence | 否，必须传入threshold | +| nan值/极大值告警 |根据是否提供threshold来判断nan值或极大值| AnomalyNan | 是， 若未配置args或未配置threshold，则默认检测nan，若提供threshold，则检测nan值以及绝对值超过阈值的极大值 | + +除此之外，我们在alert中支持dump配置项，如果打开"`dump`"选项，则会将异常信息落盘到目录`monitor_output/anomaly_detected`。 + +- 历史均值偏离告警案例如下： ```json "alert": { "rules": [{"rule_name": "AnomalyTurbulence", "args": {"threshold": 0.5}}], "dump": true }, ``` +- nan值/极大值告警案例如下： +```json + "alert": { + "rules": [{"rule_name": "AnomalyNan", "args": {"threshold": 1e10}}], + "dump": true + }, +``` + 2. 实例化工具时传入流水线并行group ```python monitor = TrainerMon( @@ -395,9 +420,9 @@ python3 -m msprobe.pytorch.monitor.anomaly_analyse -d $MONITOR_OUTPUT_DIR/anomal ``` 异常事件分析结束，将topk事件写入文件`anomaly_detected/anomaly_analyse.json`。异常分析支持以下参数配置： -| 字段名 | 解释 | 是否必选 | -| ----------------- | ------------------------------------------------------------ | -------- | -| -d 或 --data_path | 指定梯度异常落盘文件夹，梯度监控功能输出，一般为$MONITOR_OUTPUT_DIR/anomaly_detected。 | 是 | +| 字段名 | 解释 | 是否必选 | +| ----------------- | --------------------------------------------------------- | -------- | +| -d 或 --data_path | 指定异常落盘文件夹，监控功能输出，一般为$MONITOR_OUTPUT_DIR/anomaly_detected。 | 是 | | -o 或 --out_path | 排序后的异常落盘文件地址，默认在--data_path路径下落盘一个anomaly_analyse.json文件。 | 否 | | -k 或 --topk | 指定保留前topk个异常，默认为8。 | 否 | | -s 或 --step_list | 指定分析的step范围，默认为[]。 | 否 | @@ -412,37 +437,46 @@ from msprobe.pytorch.monitor.csv2tb import csv2tensorboard_by_step # 前三个参数用来指定需要转换的一批文件，指定monitor输出目录及一个时间范围，会对这个范围内的文件进行转换 # process_num指定拉起的进程个数，默认为1，更多的进程个数可以加速转换 # data_type_list是一个列表，指定需要转换的数据类型，默认转换全部数据，数据类型应来自输出件文件前缀，所有类型数据： -# ["actv", "actv_grad", "exp_avg", "exp_avg_sq", "grad_unreduced", "grad_reduced", "param"] +# ["actv", "actv_grad", "exp_avg", "exp_avg_sq", "grad_unreduced", "grad_reduced", "param_origin", "param_updated"] # output_dirpath可指定输出目录，默认保存到"{curtime}_csv2tensorboard_by_step"文件夹，其中curtime为自动获取的当前时间戳 csv2tensorboard_by_step( monitor_path="~/monitor_output", # 必填 time_start="Dec03_21-34-40", # 必填 time_end="Dec03_21-34-42", # 必填 process_num=8, - data_type_list=["param"] + data_type_list=["param_origin"] ) ``` ### 动态启停 动态启停模式：支持用户在训练过程中随时启动/更新监控。 -用户可在训练开始前通过配置环境变量DYNAMIC_MONITOR=True来确认开启动态启停模式，该模式下需要配合config.json文件中的dynamic_on字段来使用。 +用户可在训练开始前通过配置环境变量`DYNAMIC_MONITOR=True`来确认进入动态启停模式，该模式下需要配合config.json文件中的`dynamic_on`字段来使用。 在动态启停模式下，启动和停止分别由如下控制： -- 启动： - 首次监控：config.json文件中dynamic_on字段为true，代表是否需要开启监控。 - 非首次监控：config文件时间戳更新且config.json文件中dynamic_on字段为true。 -- 停止： - 到达collect_times之后自动停止并改config.json文件中dynamic_on字段为false，可再通过上述操作重启。 +- **启动**： + - 首次监控：查看config.json文件中`dynamic_on`字段，若为`true`则在下一步开启监控。 + - 非首次监控：查看config.json文件时间戳，若时间戳更新且config.json文件中`dynamic_on`字段为`true`则在下一步开启监控。 +- **停止**： + 到达`collect_times`之后自动停止并改config.json文件中`dynamic_on`字段为`false`，可再通过上述操作重启。 -大部分情况下，用户可在看到异常趋势后再手动更新config.json文件并打开dynamic_on开关；此外，使用时若想要在一开始就启动监控，可直接打开dynamic_on开关做基础配置的监测（首次不要求时间戳更新） +**注意事项：**： -注意事项： +- 默认监控启动皆统一在配置初始化或查询到更新后的下一步，即第n步挂上hook将在第n+1步启动采集，如需采集第0步数据请使用静态模式。 +- config.json中途修改出错时，若此时不在监控则不生效，若在监控则用原配置继续。 +- 达到`collect_times`之后程序会自动将该值置为`false`待下次改`true`重启。 -- 默认监控启动皆统一在配置初始化或查询到更新后的下一步，也就是若第n步挂上hook则第n+1步才启动采集，如需采集第0步数据请用静态模式。 -- config中途修改出错时，若此时不在监控就不生效，若在监控则用原配置继续。 -- 达到collect_times之后会自动将该值置为false待下次改true重启。 +**支持的使用场景说明如下：** + +| 场景 | 监控模式 | 操作步骤 | 结果描述 | +|-----------------------------------------------|----|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------| +| 场景1: 使用默认静态模式 | 静态 | 1. 配置环境变量：`export DYNAMIC_MONITOR=False `
或不设置该环境变量 | 走默认分支进行数据采集和保存，不受config.json中`dynamic_on`影响 | +| 场景2: 进入动态启停模式，初始不启动监控 | 动态 | 1.配置环境变量：`export DYNAMIC_MONITOR=True`
2.配置config.json中`dynamic_on: false`或不设置该字段 | 初始状态下无监控，不进行数据采集和保存 | +| 场景3: 进入动态启停模式，初始即启动监控 | 动态 | 1.配置环境变量：`export DYNAMIC_MONITOR=True`
2.配置config.json中`dynamic_on: true` | 根据初始配置在第1步（初始计数为0）开启监控并保存，采集`collect_times`次数后结束监控 | +| 场景4: 进入动态启停模式，初始暂不启动监控，训练中途启动 | 动态 | 1.配置环境变量：`export DYNAMIC_MONITOR=True`
2.开始时配置config.json中`dynamic_on: false`或不设置该字段
3.训练中途修改config.json中`dynamic_on: true` | 训练中途根据最新配置在下一步开启监控并保存，采集`collect_times`次数后结束监控 | +| 场景5: 进入动态启停模式，监控还未结束时中途修改config.json采集配置 | 动态 | 1.配置环境变量：`export DYNAMIC_MONITOR=True`
2.期间配置`dynamic_on: true`启动采集
3.在采集还未达到`collect_times`次数前，中途修改config.json配置 | 更新前按旧配置采集并保存，更新后下一步以最新config.json采集且`collect_times`重新从0开始计数。此功能可配合中途`collect_times`改0来实现提前停止监控。 +| 场景6: 进入动态启停模式，在根据`collect_times`结束监控后，需重新启动监控 | 动态 | 1.配置环境变量：`export DYNAMIC_MONITOR=True`
2.期间`dynamic_on: true`启动采集
3.采集达到`collect_times`次数后结束监控，程序自动改`dynamic_on:false`
4.配置config.json中`dynamic_on:true`重启监控 | 更新前按旧配置采集并保存，中途停止监控后无采集，重启后下一步以最新config.json重启采集且`collect_times`重新从0开始计数。 ### 功能重载 此功能将在2026年废弃。请使用[动态启停](#动态启停)功能代替。 @@ -498,7 +532,7 @@ csv2tensorboard_by_step(monitor_path, time_start, time_end, process_num=1, data_ | time_start | 起始时间戳。搭配time_end一起使用。指定一个时间范围，会对这个范围内的文件进行转换。左闭右闭的区间。 | 是 | | time_end | 结束时间戳。搭配time_start一起使用。指定一个时间范围，会对这个范围内的文件进行转换。左闭右闭的区间。 | 是 | | process_num | 指定拉起的进程个数，默认为1，更多的进程个数可以加速转换。 | 否 | -| data_type_list | 指定需要转换的数据类型, 数据类型应来自输出件文件前缀，所有类型数据：
["actv", "actv_grad", "exp_avg", "exp_avg_sq", "grad_unreduced", "grad_reduced", "param"]。
不指定就转换全部数据。 | 否 | +| data_type_list | 指定需要转换的数据类型, 数据类型应来自输出件文件前缀，所有类型数据：
["actv", "actv_grad", "exp_avg", "exp_avg_sq", "grad_unreduced", "grad_reduced", "param_origin", "param_updated"]。
不指定就转换全部数据。 | 否 | | output_dirpath | 指定转换后的输出路径，默认输出到"{curtime}_csv2tensorboard_by_step"文件夹，其中curtime为自动获取的当前时间戳。 | 否 | - 在模型任意位置获取当前参数**梯度**统计量 ```python @@ -577,33 +611,35 @@ TrainerMon.monitor_gnorm_with_ad(model, grad_acc_steps, optimizer, dp_group, tp_ 下面详细解释各个字段： -| 字段名字 | 是否必选 | 解释 | -| ----------------------- | -------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| "targets" | 可选 | 指定需要监控的模型层和监控对象， 例如transformer的第0层language_model.encoder.layers.0，可选择监控input、output、input_grad、output_grad。如果不清楚模型结构， 可以将 "print_struct" 字段设置为 true， 监控工具会打印模型中torch module的名字和详细结构，并在第1个step后退出。未配置时默认为全量监控。 | -| "input" | 可选 | "tuple[2]:0"的意思是目标module的前向input参数为长度为2的tuple， 我们关心的是tuple第0个元素。 | -| "output" | 必选 | "tensor"的意思是目标module的前向output参数类型为tensor | -| "input_grad" | 可选 | "tuple[2]:0"的意思是目标module的后向input_grad参数是长度为2的tuple， 我们关心的是tuple的第0个元素。 | -| "output_grad" | 必选 | "tuple[1]:0"的意思是目标module的后向input_grad参数是长度为1的tuple， 我们关心的是tuple的第0个元素。 | -| "dynamic_on" | 可选 | 在动态启停时使用，true代表打开监控，false代表关闭监控，默认值为false，且达到collect_times之后会自动将该值置为false待下次改true重启。**仅PyTorch场景支持此参数**。 | -| "collect_times" | 可选 | 设置采集次数，达到该次数后停止监控，默认值为100000000，目的是一直采集。 | -| "start_step" | 可选 | 设置开始采集step，模型训练达到start_step后开始监控采集，默认值为0，表示从step0开始监控采集。 | -| "step_interval" | 可选 | 设置采集step间隔，默认值为1，表示每个step均采集监控数据。 | -| "print_struct" | 可选 | 设置为true后监控工具会打印模型中torch module的名字和详细结构，并在第1个step后退出。不填默认为false。**仅PyTorch场景支持此参数**。 | -| "module_ranks" | 可选 | 用于在分布式训练场景中希望控制在哪些rank开启module监控。如果不填，则默认在所有rank开启。 列表内rank要求为int类型。 | -| "ur_distribution" | 可选 | 若为true则会统计adam优化器指定模块（targets中指定）参数的update和ratio向量的数值分布，并展示在heatmap里，默认为false，同时format字段必须设置为tensorboard。
依赖histc算子， 需要CANN8.0.rc2以上版本， 否则会有严重的性能问题。**仅PyTorch场景支持此参数**。 | -| "xy_distribution" | 可选 | 若为true则会监控指定module（targets中指定）的输入输出张量。 默认为false。 | -| "all_xy" | 可选 | 开启xy_distribution后生效，若为true，监控所有module。默认为false。
与targets同时生效，all_xy配置为true时，若targets配置module_xx和指定对象，则module_xx按targets配置生效，其他module则监控全部对象，包含input、output、input_grad、output_grad。 | -| "forward_only" | 可选 | 开启xy_distribution后生效，若为true，仅监控指定module的前向，targets中的input_grad、output_grad不生效。默认为false。 | -| "backward_only" | 可选 | 开启xy_distribution后生效，若为true，仅监控指定module的反向，targets中的input、output不生效。默认为false。 | -| "mv_distribution" | 可选 | 若为true则会监控指定模块中的参数的优化器状态， 默认为false。版本依赖histc算子， 需要CANN8.0.rc2以上版本， 否则会有严重的性能问题。**仅PyTorch场景支持此参数**。 | +| "xy_distribution" | 可选 | 若为true则会监控指定module（targets中指定）的输入输出张量。 默认为false。 | +| "all_xy" | 可选 | 开启xy_distribution后生效，若为true，监控所有module。默认为false。
与targets同时生效，all_xy配置为true时，若targets配置module_xx和指定对象，则module_xx按targets配置生效，其他module则监控全部对象，包含input、output、input_grad、output_grad。 | +| "forward_only" | 可选 | 开启xy_distribution后生效，若为true，仅监控指定module的前向，targets中的input_grad、output_grad不生效。默认为false。 | +| "backward_only" | 可选 | 开启xy_distribution后生效，若为true，仅监控指定module的反向，targets中的input、output不生效。默认为false。 | +| "mv_distribution" | 可选 | 若为true则会监控指定模块中的参数的优化器状态， 默认为false。版本=2.4.0 -## 展示示例 +## 更新通知 -支持重建模型的层级结构； +请注意，tb_graph_ascend插件已于2025/3/12更新到1.0.0版本，如果当前环境已安装旧版本插件，推荐升级。 -支持两个模型的结构差异比对； +更新内容如下： -支持两个模型的精度数据比对，支持疑似有精度问题节点的快速搜索，自动跳转展开节点所在的层级。 +- 优化了信息栏，使用了更人性化、更美观的展示界面； +- 提升了节点渲染和搜索性能； +- 双图比对场景画布分离，操作左图时不会影响到右图； +- 新增浏览器匹配节点功能，双图比对场景有未匹配节点时，可通过在浏览器页面手动选中调试侧和标杆侧的未匹配节点进行精度比对； +- 新增颜色图例可配置功能。 + +## 工具特性 + +- 支持重建模型的层级结构； +- 支持两个模型的结构差异比对； +- 支持两个模型的精度数据比对； +- 支持模型数据的溢出检测； +- 支持多卡场景的批量构图，能够关联各卡的通信节点，分析各卡之间的数据传递； +- 支持节点名称搜索，按精度比对结果筛选节点，按溢出检测结果筛选节点，支持自动跳转展开节点所在的层级； +- 支持跨套件、跨框架的模型比对。 ![vis_show](./img/visualization/vis_showcase.png) ## 1.依赖安装 -分级可视化工具依赖**msprobe工具**和**tensorboard。** - ### 1.1 安装msprobe工具 [msprobe工具安装](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/01.installation.md) @@ -28,6 +40,8 @@ ``pip3 install tb-graph-ascend``即可。 +如需升级工具，请先``pip3 uninstall tb-graph-ascend``再``pip3 install tb-graph-ascend``即可。 + ## 2.模型结构数据采集 [MindSpore场景的精度数据采集](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md) @@ -47,7 +61,7 @@ msprobe -f mindspore graph -i ./compare.json -o ./output |-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | | -i 或 --input_path | 指定比对文件，参考[比对文件说明](#313-比对文件说明) | 是 | | -o 或 --output_path | 配置比对结果文件存盘目录，str 类型。文件名称基于时间戳自动生成，格式为：`compare_{timestamp}.vis或build_{timestamp}.vis`。 | 是 | -| -lm 或 --layer_mapping| 跨框架比对，MindSpore和PyTorch的比对场景。配置该参数时表示开启跨框架Layer层的比对功能，指定模型代码中的Layer层后，可以识别对应dump数据中的模块或API。需要指定自定义映射文件*.yaml。自定义映射文件的格式请参见[自定义映射文件（Layer）](#71-自定义映射文件layer), 如何配置自定义映射文件请参考[模型分级可视化如何配置layer mapping映射文件](./visualization/layer_mapping_example.md)。 | 否 | +| -lm 或 --layer_mapping| 跨框架比对，MindSpore和PyTorch的比对场景。配置该参数时表示开启跨框架Layer层的比对功能，指定模型代码中的Layer层后，可以识别对应dump数据中的模块或API。需要指定自定义映射文件*.yaml。自定义映射文件的格式请参见[自定义映射文件（Layer）](#71-自定义映射文件layer), 如何配置自定义映射文件请参考[模型分级可视化如何配置layer mapping映射文件](./visualization/layer_mapping_example.md)。配置该参数后，将仅按节点名称进行比对，忽略节点的 type 和 shape。如果调试侧和标杆侧有名称不同的节点，则需要配置自定义映射文件，-lm参数传入自定义映射文件路径；如果调试侧和标杆侧节点名称相同，则仅指定-lm即可。 | 否 | | -oc 或 --overflow_check | 是否开启溢出检测模式，开启后会在输出vis文件中（`compare_{timestamp}.vis或build_{timestamp}.vis`）对每个溢出节点进行标记溢出等级，溢出等级说明参考[溢出等级说明](#312-溢出等级说明) | 否 | | -f 或 --fuzzy_match | 是否开启模糊匹配，bool类型。模糊匹配说明参考[匹配说明](#311-匹配说明) | 否 | | -cs 或 --complete_stack | 是否使用完整的堆栈信息，bool类型。默认使用精简的堆栈信息，数据量小有助于增加流畅度。完整堆栈和精简堆栈信息参考[堆栈信息说明](#72-堆栈信息说明) | 否 | @@ -62,7 +76,7 @@ msprobe -f mindspore graph -i ./compare.json -o ./output - 节点的层级一致（父节点们一致） 2.模糊匹配 -- Cell节点dump名称一致，两个匹配上的Cell节点, 忽略各自节点下所有api的dump调用次数，按照名称一致+Cell节点内的调用顺序进行匹配 +- Cell节点dump名称一致，两个匹配上的Cell节点，忽略各自节点下所有api的dump调用次数，按照名称一致+Cell节点内的调用顺序进行匹配 - ![fuzzy_match_ms.png](./img/visualization/fuzzy_match_ms.png) - 参数shape一致 @@ -83,11 +97,11 @@ msprobe -f mindspore graph -i ./compare.json -o ./output ``` **比对文件参数说明**： -| 参数名 | 说明 | 是否必选 | -|-------------------|-------------------------------------------------------------------------------------------------------|------| -| npu_path | 指定待调试侧比对路径，str类型。工具根据路径格式自动进行单rank比对、多rank批量比对或多step批量比对，具体格式参考3.2 图构建和比对。 | 是 | -| bench_path | 指定标杆侧比对路径，str类型。单图构建场景可以不配置 | 否 | -| is_print_compare_log | 配置是否开启单个算子的日志打屏。可取值 true 或 false，默认为 true。关闭后则只输出常规日志，bool 类型。 | 否 | +| 参数名 | 说明 | 是否必选 | +|-------------------|----------------------------------------------------------------------------|------| +| npu_path | 指定待调试侧比对路径，str类型。工具根据路径格式自动进行单rank比对、多rank批量比对或多step批量比对，具体格式参考3.2 图构建和比对。 | 是 | +| bench_path | 指定标杆侧比对路径，str类型。单图构建场景可以不配置。 | 否 | +| is_print_compare_log | 配置是否开启单个算子的日志打屏。可取值 true 或 false，默认为 true。关闭后则只输出常规日志，bool 类型。 | 否 | ### 3.2 图构建和比对 @@ -329,11 +343,25 @@ tensorboard --logdir out_path --bind_all --port [可选，端口号] ubuntu是机器地址，6008是端口号。 -**注意，ubuntu需要替换为真实的服务器地址，例如真实的服务器地址为10.123.456.78，则需要在浏览器窗口输入http://10.123.456.78:6008** +**注意，ubuntu需要替换为真实的服务器地址，例如真实的服务器地址为10.123.456.78，则需要在浏览器窗口输入 http://10.123.456.78:6008** ### 4.2 不可直连的服务器 -**如果链接打不开（服务器无法直连需要挂vpn才能连接等场景），可以尝试使用vscode连接服务器，在vscode终端输入：** +**如果链接打不开（服务器无法直连需要挂vpn才能连接等场景），可以尝试以下方法，选择其一即可：** +1.本地电脑网络手动设置代理，例如Windows10系统，在【手动设置代理】中添加服务器地址（例如10.123.456.78） + +![proxy](./img/visualization/proxy.png) + +然后，在服务器中输入： +``` +tensorboard --logdir out_path --bind_all --port 6008[可选，端口号] +``` + +最后，在浏览器窗口输入 http://10.123.456.78:6008 + +**注意，如果当前服务器开启了防火墙，则此方法无效，需要关闭防火墙，或者尝试后续方法** + +2.或者使用vscode连接服务器，在vscode终端输入： ``` tensorboard --logdir out_path ``` @@ -341,6 +369,14 @@ tensorboard --logdir out_path 按住CTRL点击链接即可 +3.或者将构图结果件vis文件从服务器传输至本地电脑，在本地电脑中安装tb_graph_ascend插件查看构图结果 + +电脑终端输入： +``` +tensorboard --logdir out_path +``` +按住CTRL点击链接即可 + ## 5.浏览器查看 ### 5.1 浏览器打开图 @@ -359,37 +395,69 @@ tensorboard --logdir out_path ![vis_precision_info.png](./img/visualization/vis_precision_info.png) ### 5.5 未匹配节点筛选 -节点匹配规则： -1.名称一致 +参考[匹配说明](#311-匹配说明) ，不符合匹配规则的节点为无匹配节点，颜色标灰。适用于排查两个模型结构差异的场景。 -2.节点输入输出参数数量一致，参数type、shape一致 +![vis_unmatch_info.png](./img/visualization/vis_unmatch_info.png) -3.节点的层级一致（父节点们一致） +### 5.6 手动选择节点匹配 -![vis_unmatch_info.png](./img/visualization/vis_unmatch_info.png) +可通过浏览器界面，通过鼠标选择两个待匹配的灰色节点进行匹配。当前暂不支持真实数据模式。 + +![vis_match_info.png](./img/visualization/vis_match_info.png) ## 6.图比对说明 -### 颜色 +### 6.1 颜色 颜色越深，精度比对差异越大，越可疑，具体信息可见浏览器页面左下角颜色图例。 -### 疑似有精度问题判定 - -#### 真实数据模式 +#### 6.1.1 真实数据模式 节点中所有输入的最小双千指标和所有输出的最小双千分之一指标的差值，反映了双千指标的下降情况，**值越大精度差距越大，颜色标记越深**。 ``One Thousandth Err Ratio（双千分之一）精度指标：Tensor中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的比例占总元素个数的比例，比例越接近1越好`` -#### 统计信息模式 +如果调试侧（NPU）节点的output指标中的最大值（MAX）或最小值（MIN）中存在 nan/inf/-inf，直接标记为最深颜色。 + +#### 6.1.2 统计信息模式 节点中输出的统计量相对误差，**值越大精度差距越大，颜色标记越深**。 -``相对误差：abs（(npu统计值 - bench统计值) / bench统计值)`` +``相对误差：abs((npu统计值 - bench统计值) / bench统计值)`` -#### md5模式 +如果调试侧（NPU）节点的output指标中的最大值（MAX）或最小值（MIN）中存在 nan/inf/-inf，直接标记为最深颜色。 + +#### 6.1.3 md5模式 节点中任意输入输出的md5值不同。 +### 6.2 指标说明 + +精度比对从三个层面评估 API 的精度，依次是：真实数据模式、统计数据模式和 MD5 模式。比对结果分别有不同的指标。 + +**公共指标**： +- name: 参数名称，例如input.0 +- type: 类型，例如mindspore.Tensor +- dtype: 数据类型，例如BFloat32 +- shape: 张量形状，例如[32, 1, 32] +- Max: 最大值 +- Min: 最小值 +- Mean: 平均值 +- Norm: L2-范数 + +**真实数据模式指标**： +- Cosine: tensor 余弦相似度 +- EucDist: tensor 欧式距离 +- MaxAbsErr: tensor 最大绝对误差 +- MaxRelativeErr: tensor 最大相对误差 +- One Thousandth Err Ratio: tensor 相对误差小于千分之一的比例（双千分之一） +- Five Thousandth Err Ratio: tensor 相对误差小于千分之五的比例（双千分之五） + +**统计数据模式指标** +- (Max, Min, Mean, Norm) diff: 统计量绝对误差 +- (Max, Min, Mean, Norm) RelativeErr: 统计量相对误差 + +**MD5模式指标** +- md5: CRC-32 值 + ## 7.附录 ### 7.1 自定义映射文件（Layer） @@ -488,5 +556,9 @@ yaml文件中只需配置MindSpore与PyTorch模型代码中功能一致但名称 节点呈现灰色，代表左边待调试侧节点与右边标杆侧节点没有匹配上，可能有以下几点原因： - **标杆侧确实没有能与待调试侧匹配上的节点**，属于代码实现上的差异，请确认此差异是否正常，是否会影响到整网精度。 -- **节点的输入或输出type、shape不一致，参数个数不一致，节点所在层级的父层级不一致**，导致节点无法匹配，具体匹配规则见[匹配说明](#311-匹配说明)，可尝试使用模糊匹配功能，如何使用此功能请参考[构图命令行说明](#31-构图命令行说明)。如果是参数shape不一致，即使是模糊匹配功能也无法让节点匹配上，请检查参数shape不一致是否合理。 -- **节点名称不一致**，导致节点无法匹配，可使用layer mapping功能，如何使用此功能请参考[构图命令行说明](#31-构图命令行说明)，如何自定义映射文件请参考[模型分级可视化如何配置layer mapping映射文件](./visualization/layer_mapping_example.md)。 +- **节点名称一致，但节点的输入或输出type、shape不一致，参数个数不一致，节点所在层级的父层级不一致，导致节点无法匹配** + - 具体匹配规则见[匹配说明](#311-匹配说明)，可尝试使用模糊匹配功能，如何使用此功能请参考[构图命令行说明](#31-构图命令行说明)； + - 如果是参数shape不一致，即使是模糊匹配功能也无法让节点匹配上，请检查参数shape不一致是否合理。 +- **节点名称不一致**，导致节点无法匹配，目前提供两种方法，选其一即可 + - 可使用layer mapping功能，如何使用此功能请参考[构图命令行说明](#31-构图命令行说明)，如何自定义映射文件请参考[模型分级可视化如何配置layer mapping映射文件](./visualization/layer_mapping_example.md)； + - 可通过浏览器页面手动选择未匹配节点进行匹配，请参考[手动选择节点匹配](#56-手动选择节点匹配)。 diff --git a/debug/accuracy_tools/msprobe/docs/23.generate_operator_PyTorch.md b/debug/accuracy_tools/msprobe/docs/23.generate_operator_PyTorch.md index 59e2755ec3e5a3939af3a20d19fda12031a9bf51..e7c8dc7de74930d6ef9c5ef2c172a9dda4d4a040 100644 --- a/debug/accuracy_tools/msprobe/docs/23.generate_operator_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/23.generate_operator_PyTorch.md @@ -33,15 +33,15 @@ b. 在生成单API脚本时可以选择由工具构造随机数获得 dump 数 ``` **配置文件参数说明** - | 参数名称 | 解释 | 是否必选 | - | ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | - | dump_json_path | dump.json的文件路径，包含所有dump算子的信息；如果已经提取了可疑算子并保存可以不指定。 | 否 | - | api_name | 算子名，如Functional.softmax.3、Tensor.add.0、Torch.matmul.5等。如果已经提取了可疑算子并保存可以不指定 | 否 | - | extract_api_path | 提取可疑算子的json文件路径 | 是 | - | propagation | 选择复现算子的forward还是backward，默认为forward | 否 | - | data_mode | 选择复现算子的随机数据（random_data）还是真实数据（real_data）模式，默认为random_data | 否 | - | random_seed | 仅random_data模式有效，表示手动设定的随机种子，默认为1234 | 否 | - | iter_times | 仅random_data模式有效，表示单API运行的次数 | 否 | + | 参数名称 | 解释 | 是否必选 | + | ---------------------------- |----------------------------------------------------------------------------| ---------------------------------- | + | dump_json_path | dump.json的文件路径，包含所有dump算子的信息；如果已经提取了可疑算子并保存可以不指定。 | 否 | + | api_name | 算子名，如Functional.softmax.3、Tensor.add.0、Torch.matmul.5等。如果已经提取了可疑算子并保存可以不指定 | 否 | + | extract_api_path | 提取可疑算子的json文件路径 | 是 | + | propagation | 选择复现算子的forward还是backward，默认为forward | 否 | + | data_mode | 选择复现算子的随机数据（random_data）还是真实数据（real_data）模式，默认为random_data | 否 | + | random_seed | 仅random_data模式有效，表示手动设定的随机种子，默认为1234 | 否 | + | iter_times | 仅random_data模式有效，表示单API运行的次数，由于安全相关原因，最大支持设置为1000 | 否 | ### 2.2 运行命令生成单API脚本 config_op.json配置好后，运行如下命令： diff --git a/debug/accuracy_tools/msprobe/docs/25.tool_function_introduction.md b/debug/accuracy_tools/msprobe/docs/25.tool_function_introduction.md index f6f5db9781223fc299df978dfd55a9d2af2e07e6..d565741e9e9f5aee92269cf1e9092ea0a63063c0 100644 --- a/debug/accuracy_tools/msprobe/docs/25.tool_function_introduction.md +++ b/debug/accuracy_tools/msprobe/docs/25.tool_function_introduction.md @@ -9,7 +9,7 @@ | [整网比对
（compare）](./10.accuracy_compare_PyTorch.md) | 计算模型整网NPU和标杆设备的精度误差指标，标记精度异常API或Module，助力快速定位精度问题根因。 | 1、整网比对定位精度可疑算子 | 1、由于使用整网dump数据，定位的可疑算子受累计误差影响
2、当模型规模较大时，比对所需时间较长 | | [在线预检
（online_api_accuracy_checker）](./08.accuracy_checker_online_PyTorch.md) | 通过TCP通信或共享存储空间的方式，进行在线精度预检，解决离线预检大数据量落盘、传输困难痛点。 | 1、使用离线预检，数据量较大落盘困难或传输耗时长时，可通过在线预检进行精度排查 | 1、依赖GPU环境，NPU和GPU能够通信
2、重计算模式下，不支持反向aten算子预检 | | [溢出检查
（overflow_checker）](./12.overflow_check_PyTorch.md) | 检测模型计算过程的输入输出，并在溢出时落盘数据，助力用户快速定位溢出位置。 | 1、当模型出现溢出时，用于快速定位最先溢出的API或Module
2、相比数据采集，性能更优，磁盘压力更小 | 1、局限性同数据采集 | -| [数据解析
（parse_tool）](./14.data_parse_PyTorch.md) | 互交式界面处理解析kernel层级dump数据，便于查看分析。 | 1、比对kernel层级dump数据的一致性 | 1、仅限于NPU | +| [数据解析
（parse_tool）](./14.data_parse_PyTorch.md) | 交互式界面处理解析kernel层级dump数据，便于查看分析。 | 1、比对kernel层级dump数据的一致性 | 1、仅限于NPU | | [无标杆比对
（free_benchmark）](./15.free_benchmarking_PyTorch.md) | 不依赖标杆数据，通过对算子输入增加微小扰动，计算扰动后输出与原始输出的相对误差，识别有精度风险算子。 | 1、无标杆数据场景下的算子精度排查
2、对个别算子进行升精度、“to cpu”等操作，以验证其对模型loss的影响 | 1、由于需要拷贝输入进行二次执行，所以在遇到大张量的输入时容易发生显存OOM的问题, 特别是反向比对过程。建议结合白名单使用
2、比对会延长训练时间，整网比对可能会造成严重的耗时膨胀，建议结合白名单使用 | | [梯度状态监测
（grad_probe）](./17.grad_probe.md) | 可导出模型权重梯度数据并对比相似度，助力确认训练过程精度问题step和反向中的异常。 | 1、需要分析梯度数据时
2、需要定位发生问题的step时 | 暂无 | | [在线精度比对
（online_dispatch）](./18.online_dispatch.md) | 训练过程中直接完成NPU和CPU的精度比对并输出比对结果。 | 1、执行一次就可获取NPU和CPU分别执行后的精度比对结果 | 暂无 | diff --git a/debug/accuracy_tools/msprobe/docs/26.data_dump_PyTorch_baseline.md b/debug/accuracy_tools/msprobe/docs/26.data_dump_PyTorch_baseline.md index 5ca199ab6171a3634af0b26844d6ba8e7d04933f..537c185a016bd583533aa831bdc04a10c6c49c96 100644 --- a/debug/accuracy_tools/msprobe/docs/26.data_dump_PyTorch_baseline.md +++ b/debug/accuracy_tools/msprobe/docs/26.data_dump_PyTorch_baseline.md @@ -1,8 +1,19 @@ # PyTorch 场景的精度数据采集基线 +## "statistics"模式（未开启md5）采集时间膨胀参考基线 + +该基线为PyTorch框架下，使用"statistics"模式采集数据性能膨胀的参考基线。本基线测试了LLAMA2-7B语言大模型在不同采集模式8卡下的时间膨胀。 + +| 采集模式 | 无工具 (耗时) | 加工具但未使能 Dump (耗时) | 加工具并使能 Dump (耗时) | +|:--------:|:--------:|:--------------------:|:------------------:| +| L0 | ≈17.4 s | ≈17.4 s （无膨胀） | ≈78.4 s （膨胀4.5倍） | +| L1 | ≈17.4 s | ≈20.7 s （膨胀1.2倍） | ≈353 s （膨胀20倍） | +| mix | ≈17.4 s | ≈20.7 s （膨胀1.2倍） | ≈430 s （膨胀24.7 倍） | + + ## "tensor"模式采集数据量参考基线 -该基线为pytorch框架下，使用"tensor"模式采集数据量参考基线。本基线测试了两个模型，分别为LLAMA2-7B和LLAMA2-13B，测试了不同采集模式下，不同global_batch_size下，单卡和8卡下，数据量的变化。 +该基线为PyTorch框架下，使用"tensor"模式采集数据量参考基线。本基线测试了两个模型，分别为LLAMA2-7B和LLAMA2-13B，测试了不同采集模式下，不同global_batch_size下，单卡和8卡下，数据量的变化。 ### LLAMA2-7B @@ -25,8 +36,8 @@ - - + + diff --git a/debug/accuracy_tools/msprobe/docs/27.dump_json_instruction.md b/debug/accuracy_tools/msprobe/docs/27.dump_json_instruction.md index bf5998bce0b4cd174b9713d9417d1afb674c2b56..bf992a02aba6c9b4c6c1d18077775c0a8f4325ea 100644 --- a/debug/accuracy_tools/msprobe/docs/27.dump_json_instruction.md +++ b/debug/accuracy_tools/msprobe/docs/27.dump_json_instruction.md @@ -268,7 +268,7 @@ dump.json文件中包含以下数据名称： mix级别的dump.json文件同时包括L0和L1级别的dump数据，文件格式与上述示例相同。 -## 2. MindSpore 场景下的 dump.json 文件 +## 2. MindSpore 场景下的 dump.json 文件 ### 2.1 L0 级别 diff --git a/debug/accuracy_tools/msprobe/docs/28.debugger_save_instruction.md b/debug/accuracy_tools/msprobe/docs/28.debugger_save_instruction.md index 6f4d519d5f61d5efaaffe54a1bde4f140b539f72..db4b20d24e3100833625d74c853bf3bdca1709cb 100644 --- a/debug/accuracy_tools/msprobe/docs/28.debugger_save_instruction.md +++ b/debug/accuracy_tools/msprobe/docs/28.debugger_save_instruction.md @@ -1,28 +1,33 @@ -# 单点保存工具 README +# 单点保存工具 ## 简介 -L0, L1, mix dump存在盲区，网络中的非api/module的输入输出不会被批量dump下来。单点保存提供类似np.save和print的功能和使用体验，可以保存指定的变量。同时针对大模型场景进行了增强，具备以下特性： +L0, L1, mix级别的dump能力存在盲区，网络中的非API或module的输入输出不会被批量dump下来。单点保存提供类似np.save和print的功能和使用体验，可以保存指定的变量。同时针对大模型场景进行了增强，具备以下特性： - 可保存变量的反向梯度结果。 - 能直接保存嵌套结构数据（如 list、dict），无需手动遍历。 -- 自动分 rank 保存。 +- 自动分 Rank 保存。 +- 可分 Step 保存数据。 - 多次调用时会自动计数。 -- 可配置保存统计值或者张量。 +- 可配置保存统计值（MindSpore静态图暂不支持）或者张量。 +- 支持异步保存。 ## 支持场景 -仅支持 PyTorch 与 MindSpore 的动态图场景。 -## 使能方式 +## 动态图场景（Pytorch&MindSpore） -### 配置文件说明 +### 使能方式 -通用配置： +#### 配置文件说明 + +通用配置 （细节详见[通用配置说明](./02.config_introduction.md#11-通用配置) ）： | 参数 | 解释 | 是否必选 | | -------- |-------------------------------------------| -------- | | task | dump 的任务类型，str 类型。 单点保存场景仅支持传入"statistics", "tensor"。 | 是 | | level | dump 级别，str 类型，根据不同级别采集不同数据。单点保存场景传入"debug"。 | 是 | -| dump_path | 设置 dump 数据目录路径，str 类型。细节详见[通用配置说明](./02.config_introduction.md#11-通用配置) | 是 | -| rank | 指定对某张卡上的数据进行采集，list[Union[int, str]] 类型。细节详见[通用配置说明](./02.config_introduction.md#11-通用配置) | 否 | +| dump_path | 设置 dump 数据目录路径，str 类型。 | 是 | +| rank | 指定对某张卡上的数据进行采集，list[Union[int, str]] 类型。 | 否 | +| step | 指定采集某个 Step 的数据，list[Union[int, str]] 类型。 | 否 | +| async_dump | 异步 dump 开关，bool 类型。 | 否 | "statistics" 任务子配置项： | 参数 | 解释 | 是否必选 | @@ -31,19 +36,21 @@ L0, L1, mix dump存在盲区，网络中的非api/module的输入输出不会被 "tensor" 任务无子配置项。 -### 接口调用说明 - -调用PrecisionDebugger.save，传入需要保存的变量，指定变量名称以及是否需要保存反向数据。接口入参说明详见[pytorch单点保存接口](./05.data_dump_PyTorch.md#19-save)，[mindspore单点保存接口](./06.data_dump_MindSpore.md#615-save) +#### 接口调用说明 -### 实例（以pytorch场景为例） +调用PrecisionDebugger.save，传入需要保存的变量，指定变量名称以及是否需要保存反向数据。接口入参说明详见[PyTorch单点保存接口](./05.data_dump_PyTorch.md#19-save)，[MindSpore单点保存接口](./06.data_dump_MindSpore.md#615-save) +#### 实例 +（以PyTorch场景为例，MindSpore场景只需要从msprobe.mindspore模块导包即可） 配置文件 ```json { "task": "statistics", "dump_path": "./dump_path", "rank": [], + "step": [], "level": "debug", + "async_dump": false, "statistics": { "summary_mode": "statistics" } @@ -53,7 +60,7 @@ L0, L1, mix dump存在盲区，网络中的非api/module的输入输出不会被 初始化 ```python # 训练启动py脚本 -from mindspore.pytorch import PrecisionDebugger +from msprobe.pytorch import PrecisionDebugger debugger = PrecisionDebugger("./config.json") for data, label in data_loader: # 执行模型训练 @@ -64,7 +71,7 @@ for data, label in data_loader: 初始化（无配置文件） ```python # 训练启动py脚本 -from mindspore.pytorch import PrecisionDebugger +from msprobe.pytorch import PrecisionDebugger debugger = PrecisionDebugger(dump_path="dump_path", level="debug") for data, label in data_loader: # 执行模型训练 @@ -75,20 +82,104 @@ for data, label in data_loader: 调用保存接口 ```python # 训练过程中被调用py文件 -from mindspore.pytorch import PrecisionDebugger +from msprobe.pytorch import PrecisionDebugger dict_variable = {"key1": "value1", "key2": [1, 2]} PrecisionDebugger.save(dict_variable, "dict_variable", save_backward=False) ``` +## 静态图场景（MindSpore） + +### 使能方式 + +#### 接口调用说明 +工具提供两个对外接口`save`和`save_grad`，分别用于保存训练中的tensor以及tensor对应的反向数据 +| 接口名称 | 入参 | device | MindSpore版本 |备注 | +| ------- | ------ | -------------- | --------------|--------------------------------------------------- | +| save | save_dir name, data | Ascend | >= 2.6.0 | （主流场景）图模式下只支持Ascend，pynative下支持Ascend/GPU/CPU。 | +| save_grad | save_dir, name, data | Ascend | >= 2.6.0 | （主流场景）图模式下只支持Ascend，pynative下支持Ascend/GPU/CPU。 | + + +---- +> 函数原型: +`save(save_dir:str, name:str, data)` +- save_dir：表示要保存的目录。 +- name ：表示要保存的文件标志名称。 +- data ：表示数据入参，可以是`mindspore.Tensor`或者是`List`,`Tuple`,`Dict`等嵌套结构。 + +> 函数原型： +`save_grad(save_dir:str, name:str, data)` +- save_dir：表示要保存的目录。 +- name ：表示要保存的文件标志名称。 +- data ：表示数据入参，**只能**是`mindspore.Tensor`。 + +#### 实例 + +- save接口使用： + +```python +# save api usage +# **first import** +from msprobe.mindspore import save + + +class Net(nn.Cell): + def construct(self, x, y, z): + # **use save api** + save("./test_dump", 'x', x) + return x * y * z + +x = Tensor([1, 2], ms.float32) +y = Tensor([-2, 3], ms.float32) +z = Tensor([0, 3], ms.float32) +net = Net() +output = grad(net, grad_position=(1, 2))(x, y, z) +time.sleep(1) + +# then will generate **./test_dump/step0/rank0/x_float32_0.npy** +``` + +- save_grad接口使用： + +```python +# save_grad usage +# **first import** +from msprobe.mindspore import save_grad +class Net(nn.Cell): + def construct(self, x, y, z): + # **use save api** the return value of save_grad must be received by origin + z = save_grad("./test_dump", 'z', z) + return x * y * z + +x = Tensor([1, 2], ms.float32) +y = Tensor([-2, 3], ms.float32) +z = Tensor([0, 3], ms.float32) +net = Net() +output = grad(net, grad_position=(1, 2))(x, y, z) +time.sleep(1) + +# then will generate **./test_dump/step0/rank0/z_grad_float32_0.npy** +``` +**注意**save_grad需要将返回值回传给原tensor，此操作不会有精度影响，只会传递原值。 + + ## 输出结果 +### 动态图场景（Pytorch&MindSpore） * **"task" 配置为 "statistics" 场景** ：在 dump 目录下会生成包含变量统计值信息的 `debug.json` 文件。 - * **"task" 配置为 "tensor" 场景** ：除了在 dump 目录下生成包含变量统计值信息的 `debug.json` 文件外，还会在 dump 子目录 `dump_tensor_data` 中保存张量二进制文件，文件名称格式为 `{variable_name}{grad_flag}.{count}.tensor.{indexes}.{file_suffix}`。 + `debug.json` 中统计值的key命名格式为 `{variable_name}{grad_flag}.{count}.debug`。 + * **"task" 配置为 "tensor" 场景** ：除了在 dump 目录下生成包含变量统计值信息的 `debug.json` 文件外，还会在 dump 子目录 `dump_tensor_data` 中保存张量二进制文件，文件名称格式为 `{variable_name}{grad_flag}.{count}.debug.{indexes}.{file_suffix}`。 - variable_name： 传入save接口的变量名称。 - grad_flag： 反向数据标识，反向数据为"_grad"，正向数据为""。 - count： 调用计数，多次以相同变量名称调用时的计数。 - - indexes： 索引，在保存嵌套结构数据时的索引。例如：嵌套结构为`{"key1": "value1", "key2": ["value2", "value3"]}`，"value2"的索引为"key2.0" - - file_suffix：文件后缀，pytorch场景为"pt"，mindspore场景为"npy" + - indexes： 索引，在保存嵌套结构数据时的索引。例如：嵌套结构为`{"key1": "value1", "key2": ["value2", "value3"]}`，"value2"的索引为"key2.0"。 + - file_suffix：文件后缀，PyTorch场景为"pt"，MindSpore场景为"npy"。 + +### 静态图场景（MindSpore） +在指定目录`save_dir`下生成`{step}/{rank}`目录，目录下生成指定`{name}`的npy文件，如果是save_grad接口调用，则会生成`{name}_grad`的npy文件。 + +如`save("./test_dump", 'x', x)` -> `./test_dump/step0/rank0/x_float32_0.npy`。 + +或如`z = save_grad("./test_dump", 'z', z)` -> `./test_dump/step0/rank0/z_grad_float32_0.npy`。 diff --git a/debug/accuracy_tools/msprobe/docs/29.data_dump_MSAdapter.md b/debug/accuracy_tools/msprobe/docs/29.data_dump_MSAdapter.md index f67b28af517f4552d297cf5fbe417a46bc8a6714..6549b15e7adbb8b8bc66102820672c67b0b30437 100644 --- a/debug/accuracy_tools/msprobe/docs/29.data_dump_MSAdapter.md +++ b/debug/accuracy_tools/msprobe/docs/29.data_dump_MSAdapter.md @@ -13,7 +13,7 @@ functional: # functional为算子类别，找到对应的类别，在该类别 - conv3d ``` -删除 API 的场景：部分模型代码逻辑会存在 API 原生类型校验，工具执行dump操作时，对封装后的模型 API 可能与模型的原生 API 类型不一致，此时可能引发校验失败，详见《[FAQ](FAQ.md)》中“异常情况”的第10和11条。 +删除 API 的场景：部分模型代码逻辑会存在 API 原生类型校验，工具执行dump操作时，对封装后的模型 API 可能与模型的原生 API 类型不一致，此时可能引发校验失败，详见《[FAQ](FAQ.md#33-异常情况)》中“异常情况”的第10和11条。 ## 1. 工具安装 @@ -207,7 +207,7 @@ if __name__ == "__main__": ``` * `rank`：设备 ID，每张卡的数据保存在对应的 `rank{ID}` 目录下。非分布式场景下没有 rank ID，目录名称为 rank。 * `dump_tensor_data`：保存采集到的张量数据。 -* `dump.json`： 保存 API 或 Module 前反向数据的统计量信息。包含 dump 数据的 API 名称或 Module 名称，各数据的 dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置 summary_mode="md5" 时的 CRC-32 数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#3-MSAdapter场景下的dump.json文件)。 +* `dump.json`： 保存 API 或 Module 前反向数据的统计量信息。包含 dump 数据的 API 名称或 Module 名称，各数据的 dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置 summary_mode="md5" 时的 CRC-32 数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#3-msadapter-场景下的-dumpjson-文件)。 * `stack.json`：API/Module 的调用栈信息。 * `construct.json`：分层分级结构，level 为 L1 时，construct.json 内容为空。 diff --git a/debug/accuracy_tools/msprobe/docs/30.overflow_check_MSAdapter.md b/debug/accuracy_tools/msprobe/docs/30.overflow_check_MSAdapter.md index 01d64c808d40a1e5c4ea2190c028a7c389ffbdc4..e963a60e8361be2569e7f85ee0d97df9194d6d91 100644 --- a/debug/accuracy_tools/msprobe/docs/30.overflow_check_MSAdapter.md +++ b/debug/accuracy_tools/msprobe/docs/30.overflow_check_MSAdapter.md @@ -11,9 +11,9 @@ export INF_NAN_MODE_ENABLE=1 export MS_ASCEND_CHECK_OVERFLOW_MODE="INFNAN_MODE" ``` -**a**：在处理浮点数计算溢出问题时，NPU 当前支持两种溢出模式：INF/NAN 模式与饱和模式。INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品，默认为饱和模式，且不建议使用 INF/NAN 模式；Atlas A2训练系列产品，默认为 INF/NAN 模式，且不建议使用饱和模式。对于 MindSpore 框架侧配置，仅支持对 Atlas A2 训练系列产品进行设置，默认为 INF/NAN 模式。CANN 侧 与 MindSpore 框架侧配置须一致。 +**a**：在处理浮点数计算溢出问题时，NPU 当前支持两种溢出模式：INF/NAN 模式与饱和模式。INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品，默认为饱和模式，且不支持使用 INF/NAN 模式；Atlas A2训练系列产品，默认为 INF/NAN 模式，且不建议使用饱和模式。对于 MindSpore 框架侧配置，仅支持对 Atlas A2 训练系列产品进行设置，默认为 INF/NAN 模式。CANN 侧 与 MindSpore 框架侧配置须一致。 -溢出检测任务的配置示例见["**MindSpore 动态图场景 task 配置为 overflow_check**"](./03.config_examples.md#33-task配置为overflow_check)小节。 +溢出检测任务的配置示例见["**MindSpore 动态图场景 task 配置为 overflow_check**"](./03.config_examples.md#33-task-配置为-overflow_check)小节。 ## 1 接口介绍 diff --git a/debug/accuracy_tools/msprobe/docs/31.config_check.md b/debug/accuracy_tools/msprobe/docs/31.config_check.md new file mode 100644 index 0000000000000000000000000000000000000000..4bbe9162c8e98b630887b77a0f7657ec4d686aba --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/31.config_check.md @@ -0,0 +1,95 @@ +# config check + +## 介绍 + +该工具主要适用于对比两个环境下可能影响训练精度的配置差异，支持mindspore和pytorch两个框架，包括： + +- 环境变量 +- 三方库版本 +- 训练超参 +- 权重 +- 数据集 +- 随机操作 + + +## 安装教程 + +参见 msprobe [安装教程](./01.installation.md) + +## 使用说明 + +用户需要在两个待比对的训练的环境上分别进行数据采集, 工具会采集两个环境下影响精度的配置，采集结果上传到同一机器进行比对。 + +### 数据采集 + +#### 静态数据采集 + +静态数据采集仅支持环境变量，三方库版本及训练超参采集，其中环境变量，三方库版本默认采集，训练超参采集需要用户传入启动训练的 shell 脚本路径或 yaml 配置文件， +支持多个输入，不传入表示不采集。 + +启动命令如下 +```shell +msprobe -f pytorch/mindspore config_check -d **.sh **.yaml -o output_path +``` +-f 代表训练框架，传入pytorch或mindspore，必选。 + +-d 代表数据采集模式，可传入启动训练的 shell 脚本路径或 yaml 配置文件路径，可选，不传入代表不采集。 + +-o 代表输出路径，可选，默认为 config_check_pack.zip。 + +#### 动态数据采集 + + +在训练流程执行到的第一个python脚本开始处插入如下代码： +``` +from msprobe.core.config_check import ConfigChecker +ConfigChecker.apply_patches(fmk) +``` + +说明： + +- fmk：训练框架。可选 pytorch 和 mindspore ，不传默认为 pytorch。 + +在模型初始化好之后插入如下代码： +``` +from msprobe.core.config_check import ConfigChecker +ConfigChecker(model, shell_path, output_zip_path, fmk) +``` + +说明： + +- model：初始化好的模型。不传或缺省就不会采集权重和数据集。 +- shell_path：动态采集模式下支持 **megatron** 训练超参自动捕获，使用 **megatron** 时推荐不传入，其他情况下可传入训练脚本路径，类型为列表，传入一个或多个训练配置/启动脚本。不传或缺省就不会采集超参。 +- output_zip_path：输出zip包的路径，不传默认为"./config_check_pack.zip"。 +- fmk：当前是什么框架。可选 pytorch 和 mindspore ，不传默认为 pytorch。 + +采集完成后会得到一个zip包，里面包括各项[影响精度的配置](#介绍)。会分rank和step存储，其中step为micro_step。 + +在另一个环境上执行上述操作，得到另一个zip包 + +### 数据比对 + +将两个zip包传到同一个环境下，使用如下命令进行比对： + +```shell +msprobe -f pytorch config_check -c bench_zip_path cmp_zip_path -o output_path +``` + +其中**bench_zip_path** 为标杆侧采集到的数据， **cmp_zip_path** 为待对比侧采集到的数据。 + +**output_path 会被删掉再新建**，不传默认为"./config_check_result"， 在 **output_path** 里会生成2个目录和1个文件： +- bench：bench_zip_path里打包的数据。 +- cmp：cmp_zip_path里打包的数据。 +- result.xlsx：比对结果。里面会有多个sheet页，其中**summary**总览通过情况，其余页是具体检查项的详情。其中step为micro_step。 + +## 通过标准 + +以下五项检查通过： + +- 环境变量 +- 三方库版本 +- 训练超参 +- 权重 +- 数据集 + +这五项检查在**精度比对**前必须保证达成。 diff --git a/debug/accuracy_tools/msprobe/docs/32.ckpt_compare.md b/debug/accuracy_tools/msprobe/docs/32.ckpt_compare.md new file mode 100644 index 0000000000000000000000000000000000000000..a5ae3fde39bde754243d0d6c356c71023bb63550 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/32.ckpt_compare.md @@ -0,0 +1,69 @@ +# Checkpoint Compare + +## 介绍 +在模型训练过程中或结束后，可能保存一些检查点文件(checkpoint，简称ckpt)记录当前模型、优化器等训练状态, 工具支持比较两个不同的ckpt，评估模型相似度。 + +当前支持Megatron-LM、MindSpeed(PyTorch/MindTorch)的ckpt比较。支持TP、PP、EP、VPP模型并行；支持megatron.core、megatron.legacy、TransformerEngine的模型实现。 + + +## 安装教程 + +参见 msprobe [安装教程](./01.installation.md) + +## 使用说明 +Megatron、MindSpeed的ckpt加载依赖megatron，请确保megatron在python环境中或megatron在当前路径下。 + + +启动命令如下 +```shell +msprobe --framework pytorch config_check --compare path1 path2 -o output_path.json +``` + +| 参数名 | 解释 | 是否必选 | +|--------|-------|--------| +| -f 或 --framework | 深度学习框架，str类型。比对ckpt时，当前仅支持传入pytorch。 | 是 | +| -c 或 --compare | 2个ckpt的路径 | 是 | +| -o 或 --output | 比对结果输出路径，默认为 ./ckpt_similarity.json。输出路径存在时将报错终止。 | 否 | + +Megatron-LM 和 MindSpeed 的 ckpt 目录结构如下： + +```txt +directory_name/ +├── iter_0000005/ # 某个iteration时的ckpt目录。 +│ └── mp_rank_xx_xxx/ # 单个rank的ckpt目录，xx_xxx为模型并行索引。 +│ └── model_optim_rng.pt # 包含模型参数、随机状态等的PyTorch binary文件。 +├── iter_0000010/ +├── latest_checkpointed_iteration.txt # 记录最后一个保存的ckpt的纯文本文件。 +``` + +对于--compare参数的两个路径，为directory_name时，工具通过latest_checkpointed_iteration.txt自动选择latest checkpoint进行比对. 为directory_name/iter_xxxxxxx时, 工具使用指定iteration的ckpt进行比对。暂不支持单个rank的比对。 + +## 输出示例 +Checkpoint比对结果以json文件输出，内容如下示例： +```json +{ + "decoder.layers.0.input_layernorm.weight": { + "l2": 0.0, + "cos": 0.999999, + "numel": 128, + "shape": [ + 128 + ] + }, + "decoder.layers.0.pre_mlp_layernorm.weight": { + "l2": 0.012, + "cos": 0.98, + "numel": 128, + "shape": [ + 128 + ] + } +} +``` + +统计量 | 解释 | +|-------|---------| +| l2 | 欧式距离，$\|\|a-b\|\|_2$ | +| cos | 余弦相似度， $\frac{}{\|\|a\|\|_2\|\|b\|\|_2}$ | +| numel | 参数的元素个数 | +| shape | 参数的shape | \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/docs/33.generate_operator_MindSpore.md b/debug/accuracy_tools/msprobe/docs/33.generate_operator_MindSpore.md new file mode 100644 index 0000000000000000000000000000000000000000..6d923be76a1ed0d63776456b3996a6ce37829887 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/33.generate_operator_MindSpore.md @@ -0,0 +1,109 @@ +# 单算子API自动生成脚本 + +## 1 简介 + +单算子API自动生成脚本通过提取dump数据中的可疑算子，对其进行单API复现，输出单API精度的比对结果。具体而言，该工具可以从dump数据中提取可疑API的前反向信息，根据前反向数据生成单API的前反向过程，最后通过**新精度标准比对法**a将 NPU 和 CPU 的结果进行比对，从而给出不同比对方法下的比对结果。本工具支持**随机生成模式和真实数据模式**b。 + +a. 在生成单API脚本时可以选择由工具构造随机数获得 dump 数据或选择真实输入的数据进行单API复现。随机生成模式（对应 task: "statistics"）执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式（对应 task: "tensor"）执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + +## 2 使用方式 + +### 前提 +1. 安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 +2. 已完成对训练过程的dump，获得dump.json文件。 + [MindSpore 场景下的数据采集](./06.data_dump_MindSpore.md)章节或[Msadapter 场景下的数据采集](./29.data_dump_MSAdapter.md)章节，注意需要配置 level="L1"。 + +3. 发现某个算子疑似存在精度问题，并得知算子名，如Mint.split.1、Functional.softmax.3、Tensor.add.0、Torch.matmul.5等 + +4.（可选）当需要使用Msadapter时，由于需要环境中同时存在 Torch 与 Msadapter，所以只支持在**安装原生Torch**的场景下通过export PYTHONPATH="xx/msadapter/build/lib"等通过**环境变量使能Msadapter的方式**的环境中进行预检,预检工具能够自动索引得到所需的 Torch 与 Msadapter环境,环境安装详细参考:[msadapter官网](https://gitee.com/mindspore/msadapter)。 + +### 2.1 配置config_op.json +单API复现参数配置如下（以复现softmax算子为例）： +``` +{ + "dump_json_path": "./dump.json", + "api_name": "Mint.split.1", + "extract_api_path": "Mint.split.1.json", + "propagation": "backward", + "data_mode": "random_data", + "random_seed": 42, + "iter_times": 1 +} +``` +**配置文件参数说明** + + | 参数名称 | 解释 | 是否必选 | + | ---------------------------- |----------------------------------------------------------------------------| ---------------------------------- | + | dump_json_path | dump.json的文件路径，包含所有dump算子的信息；如果已经提取了可疑算子并保存可以不指定。 | 否 | + | api_name | 算子名，如Functional.softmax.3、Tensor.add.0、Torch.matmul.5等。 | 否 | + | extract_api_path | 提取可疑算子的json文件路径 | 是 | + | propagation | 选择复现算子的forward还是backward，默认为forward | 否 | + | data_mode | 选择复现算子的随机数据（random_data）还是真实数据（real_data）模式，默认为random_data | 否 | + | random_seed | 仅random_data模式有效，表示手动设定的随机种子，默认为1234 | 否 | + | iter_times | 仅random_data模式有效，表示单API运行的次数，由于安全相关原因，最大支持设置为1000 | 否 | + + ### 2.2 运行命令生成单API脚本 +config_op.json配置好后，运行如下命令： +``` +msprobe -f mindspore op_generate -i ./config.json -o ./ +``` +或者 + +进入到mstt的generate_op_script文件夹 +``` +cd mstt/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script +``` +运行 +``` +python op_generator.py -i ./config_op.json -o ./ +``` +**参数说明** + | 参数名称 | 解释 | 是否必选 | + | ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | + | -i 或 --config_input | config_op.json的路径 | 是 | + | -o 或 --api_output_path | 单API脚本的输出路径 | 是 | + + ### 2.3 运行单API脚本 + 运行完op_generator.py后，会在指定路径下生成api_name.py的单API脚本，例如Mint.split.1.forward.py、Functional.softmax.3.backward.py、Tensor.add.0.forward.py、Torch.matmul.5.backward.py + +运行单API脚本即可获得不同比对方法下的比对结果 +``` +python api_name.py +``` + +**运行结果说明** + +单算子脚本生成到路径`./op_result_output`的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件内容详情如下： + +`accuracy_checking_details_{timestamp}.csv` + +| 字段 | 含义 | +| ------------------- | ------------------------------------------------------------ | +| API Name | API 名称。 | +| Bench Dtype | 标杆数据的 API 数据类型。 | +| Tested Dtype | 被检验数据的 API 数据类型。 | +| Shape | API 的 Shape 信息。 | +| Cosine | 被检验数据与标杆数据的余弦相似度。 | +| MaxAbsErr | 被检验数据与标杆数据的最大绝对误差。 | +| MaxRelativeErr | 被检验数据与标杆数据的最大相对误差。 | +| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过。 | +| Message | 提示信息。 | + +注意：PyTorch 无法对 dtype 为整数类型的 tensor 进行反向求导，而 MindSpore 支持。反向过程的预检仅比较 dtype 为浮点型的输出。 + +`accuracy_checking_result_{timestamp}.csv` + +| 字段 | 含义 | +| --------------------- | ----------------- | +| API Name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，error 为错误。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，error 为错误，如果是空白的话代表该 API 没有反向输出。 | +| Message | 提示信息。 | + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差判定结果决定的。具体规则详见 [4.1 API 预检指标](#41-api-预检指标)。 +需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error。 + +### 4.1 API 预检指标 + + - API 预检指标是通过对 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差的数值进行判断，得出该 API 是否符合精度标准的参考指标。 + - 余弦相似度大于 0.99，并且最大绝对误差小于 0.0001，标记“pass”，否则标记为“error”。 \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/docs/34.RL_collect.md b/debug/accuracy_tools/msprobe/docs/34.RL_collect.md new file mode 100644 index 0000000000000000000000000000000000000000..3ab9f67e1b8bfc184e5e5f8575d905002ac53cd4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/34.RL_collect.md @@ -0,0 +1,92 @@ +# 强化学习数据采集 + +## 介绍 +在强化学习训练过程中，往往存在多个模型（actor、reward、reference）和两个阶段（推理、训练），问题定界困难。 + +本工具提供一种灵活存储强化学习训练过程中关键阶段性数据的能力，并支持对比两次采集的关键数据，以支持问题快速定界。 + +常用关键数据示例：prompt、response、reward、log_prob、ref_log_probe、old_log_probe、kl_loss。 + + +## 安装教程 + +参见 msprobe [安装教程](./01.installation.md)。 + +## 使用说明 + +### 数据采集 + +用户识别脚本中需要采集数据的地方，然后通过插入代码的方式采集关键数据。 + +当确定需要采集数据的地方，例如response，可以按如下方式对数据进行存储： +``` +from msprobe.core import SingleSave +SingleSave("./dump_path", fmk="pytorch") +SingleSave.save({"response": response}) +``` +其中"./dump_path"为输出路径，没有默认值，需要自己配置；fmk可选"pytorch"或者"mindspore"，默认"pytorch"。 + +其中"response"是可以任意指定的key，response是训练过程中的真实tensor变量。 + +也支持一次性存储多个数据： +``` +from msprobe.core import SingleSave +SingleSave("./dump_path", fmk="pytorch") +SingleSave.save({ + "prompt": prompt, + "response": response + }) +``` + +### 配置保存 + +当确定需要采集数据配置json的地方，可以按如下方式对配置进行存储： +``` +from msprobe.core import SingleSave +SingleSave("./dump_path") +SingleSave.save_config(configurations_json) +``` + + +采集到的数据目录结构如下： +```txt +dump_path/ +├── data/ # 固定为data +│ └── response/ # 关键数据名称，来自SingleSave.save的时候的key +│ └── step0/ # step数 +│ └── rank0/ # rank数 +│ └── micro_step0/ #micro_step数 +| └── response0.npy #存储的关键数据的真实npy文件 +| └── response0.json #存储的关键数据的统计量文件，包括tensor的最大、最小、均值、norm、shape +├── configurations.json # 配置json文件 +``` + +### 结果比对 + +两次采集数据之后得到dump_path1和dump_path2，可以创建一个比对脚本，例如compare.py，将两次训练的dump_path传入： +``` +from msprobe.core import SingleComparator +SingleComparator.compare( + "dump_path1", + "dump_path2", + "output_path") +``` + +会在output_path下对每种关键数据都生成excel结果表格，比如response.xlsx，形式为关键数据的名字加上.xlsx后缀。 + +表格会体现每一个对应tensor的差异，解释： + +表头 | 解释 | +|-------|---------| +| step | 训练步数 | +| rank | 卡号 | +| micro_step | 梯度累计步数 | +| id | 参数的shape | +| shape1 | dump_path1中的数据形状 | +| shape2 | dump_path2中的数据形状 | +| 相同元素百分比 | 元素相同的个数占总元素个数的百分比 | +| 首个不匹配元素索引 | 首个匹配不上的元素是第几个 | +| 最大绝对误差 | 最大绝对误差 | +| 最大相对误差 | 最大相对误差 | +| 误差在千分之一内元素占比 | 误差在千分之一内元素个数占总元素个数的百分比 | +| 误差在百分之一内元素占比 | 误差在百分之一内元素个数占总元素个数的百分比 | diff --git a/debug/accuracy_tools/msprobe/docs/data_dump_MindSpore/data_dump_MindSpore_baseline.md b/debug/accuracy_tools/msprobe/docs/data_dump_MindSpore/data_dump_MindSpore_baseline.md index 0a76c51d71d77c9cbc86d98600203e6faa71a0f6..275aa66e53f25587facb2034dba5706b71bab0bb 100644 --- a/debug/accuracy_tools/msprobe/docs/data_dump_MindSpore/data_dump_MindSpore_baseline.md +++ b/debug/accuracy_tools/msprobe/docs/data_dump_MindSpore/data_dump_MindSpore_baseline.md @@ -1,6 +1,17 @@ # MindSpore 场景的精度数据采集基线 -## "tensor"模式采集数据量参考基线 +## "statistics"模式（未开启md5）采集**时间**膨胀参考基线 + +该基线为MindSpore框架下，使用"statistics"模式采集数据性能膨胀参考基线。测试了38B语言大模型在不同采集模式8卡下的性能膨胀。 + +| 采集模式 | 无工具 (耗时) | 加工具但未使能 Dump (耗时) | 加工具并使能 Dump (耗时) | +|:--------:|:-------------:|:--------------------:|:----------------:| +| L0 | ≈340 ms | ≈340 ms （无膨胀） | ≈1.2 s （膨胀3.5倍） | +| L1 | ≈340 ms | ≈0.7–1.2 s （膨胀2~4倍） | ≈3.8 s （膨胀11倍） | +| mix | ≈340 ms | ≈0.7–1.2 s （膨胀2~4倍） | ≈5.5 s （膨胀16倍） | + + +## "tensor"模式采集**数据量**参考基线 该基线为MindSpore框架下，使用"tensor"模式采集数据量参考基线。本基线测试了38B语言大模型在不同采集模式下，不同global_batch_size下，单卡和8卡下，数据量的变化。 diff --git a/debug/accuracy_tools/msprobe/docs/data_dump_MindSpore/dynamic_graph_quick_start_example.md b/debug/accuracy_tools/msprobe/docs/data_dump_MindSpore/dynamic_graph_quick_start_example.md index 543d260650361431ffb8b5142ae3df6b09d0db1d..14bb2cd2c54793b5a61af5e106bcfcd484e8ecef 100644 --- a/debug/accuracy_tools/msprobe/docs/data_dump_MindSpore/dynamic_graph_quick_start_example.md +++ b/debug/accuracy_tools/msprobe/docs/data_dump_MindSpore/dynamic_graph_quick_start_example.md @@ -51,6 +51,7 @@ debugger = PrecisionDebugger(config_path=config_path) # 设置 MindSpore 设备上下文 context.set_context(mode=ms.PYNATIVE_MODE, device_target="Ascend", device_id=0) +print("Context set successfully. Please wait for the training task.") # 定义卷积层 def conv_layer(in_channels, out_channels, kernel_size, stride=1, padding=0, pad_mode="valid", has_bias=True): @@ -199,7 +200,7 @@ python alexnet_model.py ## 5. 数据分析 -在 `dump_path` 参数指定的路径下（本例中为 `./output`），会出现如下目录结构，后续精度数据分析操作可使用 msprobe 工具的精度预检和精度比对等功能，详细流程请参见[《msprobe使用手册》](../../README.md#2-精度预检)。： +在 `dump_path` 参数指定的路径下（本例中为 `./output`），会出现如下目录结构，后续精度数据分析操作可使用 msprobe 工具的精度预检和精度比对等功能，详细流程请参见[《msprobe使用手册》](../../README.md#2-精度预检)。 ```bash output/ @@ -208,4 +209,5 @@ output/ ├── construct.json # level为L0时，保存Cell的层级关系信息。当前场景为空 ├── dump.json # 保存API前反向输入输出数据的统计量信息 └── stack.json # 保存API的调用栈 + ...... ``` \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/docs/img/compare_result.png b/debug/accuracy_tools/msprobe/docs/img/compare_result.png index b6d7ec6dfcbc44b4b7056e1297a481f495ceb86e..b321ebed8c7ea04357b57da81cc31ee038d4b94f 100644 Binary files a/debug/accuracy_tools/msprobe/docs/img/compare_result.png and b/debug/accuracy_tools/msprobe/docs/img/compare_result.png differ diff --git a/debug/accuracy_tools/msprobe/docs/img/visualization/proxy.png b/debug/accuracy_tools/msprobe/docs/img/visualization/proxy.png new file mode 100644 index 0000000000000000000000000000000000000000..3033214904ca3a8a1f50f187a382c47c23f05786 Binary files /dev/null and b/debug/accuracy_tools/msprobe/docs/img/visualization/proxy.png differ diff --git a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_browser_1.png b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_browser_1.png index 96e8521fde4b776ba915a00b5d77851b8406c153..93ee108b0cbaa145d61b75beac024dc377ecba4a 100644 Binary files a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_browser_1.png and b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_browser_1.png differ diff --git a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_match_info.png b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_match_info.png new file mode 100644 index 0000000000000000000000000000000000000000..2d0c68cd12ab31c891be6f22de04f230472d4e2d Binary files /dev/null and b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_match_info.png differ diff --git a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_precision_info.png b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_precision_info.png index ddd59b37f044fe64c02148b698b95296592e0399..5b625089d5c85b970089293ae754c3fb6488fd6d 100644 Binary files a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_precision_info.png and b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_precision_info.png differ diff --git a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_search_info.png b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_search_info.png index 7c55b33840163c388f8fde69f0bbc531b23f81f6..0db7f67f356700f55a7995b9e3c19df4de318939 100644 Binary files a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_search_info.png and b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_search_info.png differ diff --git a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_show_info.png b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_show_info.png index 9a6217e04848e671d784ed0b484d2fe10151bde7..75fb14cbdaca50d764b77696edef56d31c8cb0f9 100644 Binary files a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_show_info.png and b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_show_info.png differ diff --git a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_showcase.png b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_showcase.png index e95b5eeee663d91a67b1ace422c8681797ca96c1..f4f07dc1e7b429c862af074bf6d07ec560e788d6 100644 Binary files a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_showcase.png and b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_showcase.png differ diff --git a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_unmatch_info.png b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_unmatch_info.png index e4c9ed4306f9a7b20d031d32f18c815628030da6..4b123a4e7d06016cd76effd2cebcc30d6f4c2226 100644 Binary files a/debug/accuracy_tools/msprobe/docs/img/visualization/vis_unmatch_info.png and b/debug/accuracy_tools/msprobe/docs/img/visualization/vis_unmatch_info.png differ diff --git a/debug/accuracy_tools/msprobe/mindspore/__init__.py b/debug/accuracy_tools/msprobe/mindspore/__init__.py index cbdab34f0446ee12c07b2aba8b4f75018496eda6..5005d6921e6f56d7c869932c7f8a0ccdb019cb67 100644 --- a/debug/accuracy_tools/msprobe/mindspore/__init__.py +++ b/debug/accuracy_tools/msprobe/mindspore/__init__.py @@ -25,5 +25,4 @@ except ImportError: from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger from msprobe.mindspore.common.utils import seed_all, MsprobeStep, MsprobeInitStep from msprobe.mindspore.monitor.module_hook import TrainerMon - -os.environ["MS_HOOK_ENABLE"] = "on" +from msprobe.mindspore.dump.graph_tensor_dump import save, save_grad \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 557d731e042913da3a622035219ec8dea0409ab4..b6e4b0b0e4a59bf99c27868a3633c225fe0c3080 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -14,8 +14,10 @@ # limitations under the License. import os +from dataclasses import dataclass +from typing import Any, Optional from tqdm import tqdm - +import numpy as np from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.file_utils import FileOpen, create_directory, write_csv, load_json, load_yaml from msprobe.core.common.utils import add_time_as_suffix @@ -28,6 +30,9 @@ from msprobe.mindspore.api_accuracy_checker.utils import (check_and_get_from_jso from msprobe.mindspore.common.const import MsCompareConst from msprobe.mindspore.common.log import logger from msprobe.mindspore.api_accuracy_checker import torch_mindtorch_importer +from msprobe.core.data_dump.data_collector import build_data_collector +from msprobe.core.common.utils import Const, print_tools_ends_info, DumpPathAggregation +from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, MsCompareConst.SUPPORTED_API_LIST_FILE) @@ -59,13 +64,128 @@ class ProcessResultPacket: self.err_msg = err_msg +@dataclass +class Config: + execution_mode: str + dump_path: str + task: str + level: str + scope: Optional[Any] + list: Optional[Any] + framework: str + data_mode: str + file_format: str + dump_tensor_data_dir: str + async_dump: bool + summary_mode: Optional[Any] = None + + class ApiAccuracyChecker: def __init__(self, args): self.api_infos = dict() self.data_manager = DataManager(args.out_path, args.result_csv_path) # 在初始化时实例化 DataManager + self.save_error_data = args.save_error_data + if self.save_error_data: + config, dump_path_aggregation = self.init_save_error_data(args) + self.data_collector = build_data_collector(config) + self.data_collector.update_dump_paths(dump_path_aggregation) @staticmethod - def run_and_compare_helper(api_info, api_name_str, api_input_aggregation, forward_or_backward): + def init_save_error_data(args): + config = Config( + execution_mode="pynative", + dump_path=f"{args.out_path}", + dump_tensor_data_dir=f"{args.out_path}", + task="tensor", # 任务类型,模拟保存tensor数据 + level="L1", # 级别 + scope=None, # 作用域 (None) + list=None, # API 列表 (None) + framework=Const.MS_FRAMEWORK, # 框架类型 + data_mode="all", + file_format="npy", + async_dump=False + ) + + dump_dir = f"{args.out_path}" + dump_data_dir = os.path.join(dump_dir, "error_data") + create_directory(dump_data_dir) + dump_path_aggregation = DumpPathAggregation() + dump_path_aggregation.dump_file_path = os.path.join(dump_dir, "dump.json") + dump_path_aggregation.stack_file_path = os.path.join(dump_dir, "stack.json") + dump_path_aggregation.dump_tensor_data_dir = dump_data_dir + return config, dump_path_aggregation + + @staticmethod + def prepare_api_input_aggregation(api_info, forward_or_backward=Const.FORWARD): + """ + Args: + api_info: ApiInfo + forward_or_backward: str + Returns: + ApiInputAggregation + """ + forward_inputs = api_info.get_compute_element_list(Const.FORWARD, Const.INPUT) + kwargs = api_info.get_kwargs() + if forward_or_backward == Const.FORWARD: + gradient_inputs = None + else: + gradient_inputs = api_info.get_compute_element_list(Const.BACKWARD, Const.INPUT) + return ApiInputAggregation(forward_inputs, kwargs, gradient_inputs) + + @staticmethod + def is_api_checkable(api_name_str): + ''' + Args: + api_name_str: str, e.g. "MintFunctional.relu.0.forward", key in data field of api_info.json + Returns: + is_checkable: bool + Description: + tell whether this api is checkable based on the key in "data" dict in api_info.json + ''' + api_name_str_list = api_name_str.split(Const.SEP) + if len(api_name_str_list) < MsCompareConst.API_NAME_STR_LENGTH: + return False + api_type_str = api_name_str_list[0] + real_api_str = Const.SEP.join(api_name_str_list[1:-2]) + api_list = load_yaml(yaml_path) + supported_tensor_api_list = api_list.get(MsCompareConst.SUPPORTED_TENSOR_LIST_KEY) + supported_fusion_api_list = MsCompareConst.SUPPORTED_FUSION_LIST + if api_type_str in (MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL) \ + and global_context.get_framework() == Const.MS_FRAMEWORK: + return True + if api_type_str in MsCompareConst.MT_VALID_API_TYPES \ + and global_context.get_framework() == Const.MT_FRAMEWORK: + return True + if api_type_str == MsCompareConst.TENSOR_API and real_api_str in supported_tensor_api_list \ + and global_context.get_framework() == Const.MS_FRAMEWORK: + return True + if api_type_str == MsCompareConst.FUNCTIONAL_API and real_api_str in supported_fusion_api_list \ + and global_context.get_framework() == Const.MS_FRAMEWORK: + return True + return False + + def post_forward_hook(self, api_or_module_name, primitive_instance, args, kwargs, output): + self.data_collector.update_api_or_module_name(api_or_module_name) + module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=output) + self.data_collector.forward_data_collect_only_tensor( + api_or_module_name, + primitive_instance, + os.getpid(), + module_input_output + ) + + def backward_hook(self, api_or_module_name, module, grad_input, grad_output): + self.data_collector.update_api_or_module_name(api_or_module_name) + + module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input) + self.data_collector.backward_data_collect_only_tensor( + api_or_module_name, + module, + os.getpid(), + module_input_output + ) + + def run_and_compare_helper(self, api_info, api_name_str, api_input_aggregation, forward_or_backward): """ Args: api_info: ApiInfo @@ -83,13 +203,22 @@ class ApiAccuracyChecker: """ # get output if global_context.get_is_constructed(): - # constructed situation, need use constructed input to run mindspore api getting tested_output - tested_outputs = api_runner(api_input_aggregation, api_name_str, - forward_or_backward, global_context.get_framework()) + if forward_or_backward == Const.FORWARD: + tested_outputs, inputs, kwargs, forward_result_tuple = api_runner(api_input_aggregation, api_name_str, + forward_or_backward, + global_context.get_framework()) + elif forward_or_backward == Const.BACKWARD: + tested_outputs, gradient_inputs, backward_result_tuple = api_runner(api_input_aggregation, api_name_str, + forward_or_backward, + global_context.get_framework()) + else: + tested_outputs = api_runner(api_input_aggregation, api_name_str, + forward_or_backward, global_context.get_framework()) else: tested_outputs = api_info.get_compute_element_list(forward_or_backward, Const.OUTPUT) bench_outputs = api_runner(api_input_aggregation, api_name_str, forward_or_backward, Const.PT_FRAMEWORK) + tested_outputs = trim_output_compute_element_list(tested_outputs, forward_or_backward) bench_outputs = trim_output_compute_element_list(bench_outputs, forward_or_backward) if len(tested_outputs) != len(bench_outputs): @@ -114,64 +243,26 @@ class ApiAccuracyChecker: compare_result_dict.get(CompareConst.MAX_ABS_ERR).pass_status == CompareConst.PASS: status = CompareConst.PASS err_msg = "" + else: status = CompareConst.ERROR err_msg = (compare_result_dict.get(CompareConst.COSINE).err_msg + compare_result_dict.get(CompareConst.MAX_ABS_ERR).err_msg) + if forward_or_backward == Const.FORWARD and self.save_error_data \ + and global_context.get_is_constructed(): + api_name_str_backward = f"{api_name_str}{Const.SEP}{Const.FORWARD}" + self.post_forward_hook(api_name_str_backward, None, inputs, kwargs, forward_result_tuple) + + if forward_or_backward == Const.BACKWARD and self.save_error_data \ + and global_context.get_is_constructed(): + api_name_str_backward = f"{api_name_str}{Const.SEP}{Const.BACKWARD}" + self.backward_hook(api_name_str_backward, None, gradient_inputs, backward_result_tuple) + basic_info_status = \ BasicInfoAndStatus(api_name_with_slot, bench_dtype, tested_dtype, shape, status, err_msg) output_list.append(tuple([api_name_str, forward_or_backward, basic_info_status, compare_result_dict])) return output_list - @staticmethod - def prepare_api_input_aggregation(api_info, forward_or_backward=Const.FORWARD): - """ - Args: - api_info: ApiInfo - forward_or_backward: str - Returns: - ApiInputAggregation - """ - forward_inputs = api_info.get_compute_element_list(Const.FORWARD, Const.INPUT) - kwargs = api_info.get_kwargs() - if forward_or_backward == Const.FORWARD: - gradient_inputs = None - else: - gradient_inputs = api_info.get_compute_element_list(Const.BACKWARD, Const.INPUT) - return ApiInputAggregation(forward_inputs, kwargs, gradient_inputs) - - @staticmethod - def is_api_checkable(api_name_str): - ''' - Args: - api_name_str: str, e.g. "MintFunctional.relu.0.forward", key in data field of api_info.json - Returns: - is_checkable: bool - Description: - tell whether this api is checkable based on the key in "data" dict in api_info.json - ''' - api_name_str_list = api_name_str.split(Const.SEP) - if len(api_name_str_list) < MsCompareConst.API_NAME_STR_LENGTH: - return False - api_type_str = api_name_str_list[0] - real_api_str = Const.SEP.join(api_name_str_list[1:-2]) - api_list = load_yaml(yaml_path) - supported_tensor_api_list = api_list.get(MsCompareConst.SUPPORTED_TENSOR_LIST_KEY) - supported_fusion_api_list = MsCompareConst.SUPPORTED_FUSION_LIST - if api_type_str in (MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL) \ - and global_context.get_framework() == Const.MS_FRAMEWORK: - return True - if api_type_str in MsCompareConst.MT_VALID_API_TYPES \ - and global_context.get_framework() == Const.MT_FRAMEWORK: - return True - if api_type_str == MsCompareConst.TENSOR_API and real_api_str in supported_tensor_api_list \ - and global_context.get_framework() == Const.MS_FRAMEWORK: - return True - if api_type_str == MsCompareConst.FUNCTIONAL_API and real_api_str in supported_fusion_api_list \ - and global_context.get_framework() == Const.MS_FRAMEWORK: - return True - return False - def parse(self, api_info_path): api_info_dict = load_json(api_info_path) @@ -183,9 +274,9 @@ class ApiAccuracyChecker: MsCompareConst.TENSOR_TASK)) try: framework = check_and_get_from_json_dict(api_info_dict, MsCompareConst.FRAMEWORK, - "framework field in api_info.json", accepted_type=str, - accepted_value=(Const.MS_FRAMEWORK, - Const.MT_FRAMEWORK)) + "framework field in api_info.json", accepted_type=str, + accepted_value=(Const.MS_FRAMEWORK, + Const.MT_FRAMEWORK)) except Exception as e: framework = Const.MS_FRAMEWORK logger.warning(f"JSON parsing error in framework field: {e}") @@ -301,4 +392,4 @@ class ApiAccuracyChecker: elif process_result_packet.process_status == MsCompareConst.ProcessStatus.EXCEPTION_SKIP: self.data_manager.record_exception_skip(api_name_str, Const.BACKWARD, process_result_packet.err_msg) - self.data_manager.save_results(api_name_str) + self.data_manager.save_results(api_name_str) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index 36e506f67737cdea4452ba27f4fad0524d4c2884..e1640aab9e038d075f6879693e97352d5f3eb001 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import numpy as np import mindspore from mindspore import ops from msprobe.core.common.const import Const @@ -38,7 +40,6 @@ else: import torch - class ApiInputAggregation: def __init__(self, inputs, kwargs, gradient_inputs) -> None: """ @@ -148,13 +149,13 @@ class ApiRunner: Args: api_type_str: str, Union["MintFunctional", "Mint", "Tensor", "Functional"] api_sub_name: str, e.g. "relu" - api_platform: str: Union["mindpore", "pytorch"] + api_platform: str: Union["mindspore", "pytorch"] Return: api_instance: function object Description: - get mindspore.mint/torch api fucntion + get mindspore.mint/torch api function mindspore.mint.{api_sub_name} <--> torch.{api_sub_name} mindspore.mint.nn.functional.{api_sub_name} <--> torch.nn.functional.{api_sub_name} """ @@ -189,6 +190,8 @@ class ApiRunner: forward_result = api_instance(*inputs, **kwargs) # can be single tensor or tuple forward_result_tuple = convert_to_tuple(forward_result) res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in forward_result_tuple] + if api_platform == Const.MS_FRAMEWORK or api_platform == Const.MT_FRAMEWORK: + return res_compute_element_list, inputs, kwargs, forward_result_tuple else: if gradient_inputs is None: err_msg = f"ApiRunner.run_api failed: run backward api but gradient_inputs is missing" @@ -206,6 +209,7 @@ class ApiRunner: backward_result = grad_func(*inputs, gradient_inputs) # can be single tensor or tuple backward_result_tuple = convert_to_tuple(backward_result) res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_tuple] + return res_compute_element_list, gradient_inputs, backward_result_tuple else: # set requires_grad requires_grad_index = [] diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py index cb268efeae90a51465493c65caa948045bae4913..d885c5d95d9f10c4e9c5bb475aecbe17978cd7f6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/bench_functions/flash_attention_score.py @@ -95,6 +95,10 @@ def fusion_attention_forward(forward_params): scalar_value = forward_params.scalar_value keep_prob = forward_params.keep_prob + # 拦截 keep_prob 为 0 的情况，防止除零 + if keep_prob == 0: + raise ValueError("fusion_attention_forward: keep_prob 不能为 0，避免除零错误。") + qk = calculate_qk(q, k, attn_mask, pse, scalar_value) softmax_res, softmax_max, softmax_sum = softmax_forward(qk) if drop_mask is None or len(drop_mask.shape) == 0: @@ -115,6 +119,11 @@ def fusion_attention_backward(backward_params): pse = backward_params.pse scalar_value = backward_params.scalar_value keep_prob = backward_params.keep_prob + + # 拦截 keep_prob 为 0 的情况，防止除零 + if keep_prob == 0: + raise ValueError("fusion_attention_backward: keep_prob 不能为 0，避免除零错误。") + dp = torch.matmul(dx, v.permute(0, 1, 3, 2)) if drop_mask is None or len(drop_mask.shape) == 0: drop_res = softmax_res.permute(0, 1, 3, 2) @@ -138,34 +147,45 @@ def parse_bsnd_args(query, key, head_num, input_layout): if input_layout == "TND": raise ValueError(f"input_layout {input_layout} does not supported for now.") + + # 防止 head_num 为 0 + if n1 == 0: + raise ValueError("parse_bsnd_args: head_num (n1) 不能为 0，避免除零错误。") + try: if input_layout == "BSH": b, s1, h1 = query.shape _, s2, h2 = key.shape d = h1 // n1 + # 拦截 d 为 0 的情况 + if d == 0: + raise ValueError("parse_bsnd_args: 计算得到的 head_dim d 不能为 0。") n2 = h2 // d elif input_layout == "SBH": s1, b, h1 = query.shape s2, _, h2 = key.shape d = h1 // n1 + if d == 0: + raise ValueError("parse_bsnd_args: 计算得到的 head_dim d 不能为 0。") n2 = h2 // d elif input_layout == "BSND": b, s1, n1, d = query.shape _, s2, n2, _ = key.shape + if d == 0: + raise ValueError("parse_bsnd_args: head_dim d 不能为 0。") h1 = n1 * d h2 = n2 * d elif input_layout == "BNSD": b, n1, s1, d = query.shape _, n2, s2, _ = key.shape + if d == 0: + raise ValueError("parse_bsnd_args: head_dim d 不能为 0。") h1 = n1 * d h2 = n2 * d except Exception as e: raise ValueError(f"query.shape: {query.shape}, key.shape: {key.shape}, parse_bsnd_args error: {e}") from e - if d == 0: - raise ValueError(f"Value d must be non-zero.") - _dtype = query.dtype - ret = (b, s1, s2, n1, n2, d, h1, h2, _dtype) + ret = (b, s1, s2, n1, n2, d, h1, h2, query.dtype) return ret @@ -230,67 +250,6 @@ def convert_to_bnsd(_input, n, input_layout): return out.to(GTYPE) -def convert_from_bsnd(_input, input_layout): - """ - transform qkv from bsnd to input_layout. - B: batch_size - S: sequence_length - N: num_heads - D: head_dim - Args: - _input (torch.Tensor): tensor of shape (B,S,N,D) - input_layout (str): "BSH" or "SBH" or "BSND" or "BNSD" or "TND" - Returns: - tensor of shape (B,N,S,D) or (B,S,N,D) or (S,B,H) or (B,S,H) - """ - if input_layout == "BSH": - # (B,S,N,D)=>(B,S,N*D) - out = rearrange(_input, 'b s n d -> b s (n d)').contiguous() - elif input_layout == "SBH": - # (B,S,N,D)=>(S,B,N*D) - out = rearrange(_input, 'b s n d -> s b (n d)').contiguous() - elif input_layout == "BNSD": - # (B,S,N,D)=>(B,N,S,D) - out = rearrange(_input, 'b s n d -> b n s d').contiguous() - elif input_layout == "TND": - raise ValueError(f"input_layout {input_layout} does not supported for now.") - else: - out = _input - return out - - -def convert_to_bsnd(_input, n, input_layout): - """ - transform qkv from input_layout to bsnd. - B: batch_size - S: sequence_length - N: num_heads - D: head_dim - Args: - _input (torch.Tensor): tensor of shape (B,N,S,D) or (B,S,N,D) or (S,B,H) or (B,S,H) - n (int): num_heads - input_layout (str):"BSH" or "SBH" or "BSND" or "BNSD" or "TND" - Returns: - tensor of shape (B,S,N,D) - """ - if input_layout == "BSH": - # (B,S,N*D)=>(B,S,N,D) - out = rearrange(_input, 'b s (n d) -> b s n d', n=n) - elif input_layout == "SBH": - # (S,B,N*D)=>(B,S,N,D) - out = rearrange(_input, 's b (n d) -> b s n d', n=n) - elif input_layout == "BNSD": - # (B,N,S,D)=>(B,S,N,D) - out = rearrange(_input, 'b n s d -> b s n d', n=n) - elif input_layout == "TND": - raise ValueError(f"input_layout {input_layout} does not supported for now.") - else: - out = _input - if out.dim() != 4: - raise ValueError(f"convert qkv format failed with input_layout {input_layout}.") - return out - - def generate_attn_mask(*args): """ # 当sparse_mode=2、3、4时小算子到融合算子会走这个优化，反过来看就要拆解回原来的基本实现 @@ -417,17 +376,20 @@ def get_input_layout(*args, **kwargs): def npu_fusion_attention_forward_patch(*args, **kwargs): if len(args) < 2: - raise RuntimeError("npu_fusion_attention_forward_patch: length of args should greater than or equal to 2.") + raise RuntimeError("npu_fusion_attention_forward_patch: length of args should be greater than or equal to 2.") # query, key, value, head_num, input_layout head_num = get_head_num(*args, **kwargs) input_layout = get_input_layout(*args, **kwargs) b, s1, s2, n1, n2, d, h1, h2, dtype = parse_bsnd_args(args[0], args[1], head_num, input_layout) + # 此处 d 已在 parse_bsnd_args 中检查为非零 if n1 == n2 and s1 == s2: logger.debug(f"running case : BNSD = {b}_{n1}_{s1}_{d}, sparse = {kwargs.get('sparse_mode', 0)}") else: logger.debug(f"running case: BNSD = {b}_{n1}({n2})_{s1}({s2})_{d}, sparse = {kwargs.get('sparse_mode', 0)}") + if n2 == 0: + raise ValueError("n2 不能为 0，避免除零错误。") if not (n1 % n2 == 0 and n1 >= n2): raise ValueError(f"N1与N2不匹配,请检查: n1 = {n1}, n2 = {n2}.") @@ -436,7 +398,7 @@ def npu_fusion_attention_forward_patch(*args, **kwargs): "d": d, "h1": h1, "h2": h2, "dtype": dtype } new_kwargs = { - "keep_prob": 1, + "keep_prob": 1, # 注意：如果外部传入 keep_prob 为 0，也会在 fusion_attention_forward 中捕获 "scalar_value": kwargs.get("scalar_value", 1 / (d ** 0.5)), "sparse_mode": kwargs.get("sparse_mode", 0), "prefix": kwargs.get("prefix"), @@ -455,10 +417,13 @@ def npu_fusion_attention_backward_patch(*args, **kwargs): raise ValueError(f"Unsupported npu_fusion_attention_grad args {args}.") b, s1, s2, n1, n2, d, h1, h2, dtype = parse_bsnd_args(args[0], args[1], args[4], args[5]) + # 此处 d 已在 parse_bsnd_args 中检查为非零 if n1 == n2 and s1 == s2: logger.info(f"running case : bnsd = {b}_{n1}_{s1}_{d}, sparse = {kwargs.get('sparse_mode', 0)}") else: logger.info(f"running case: bnsd = {b}_{n1}({n2})_{s1}({s2})_{d}, sparse = {kwargs.get('sparse_mode', 0)}") + if n2 == 0: + raise ValueError("n2 不能为 0，避免除零错误。") if not (n1 % n2 == 0 and n1 >= n2): raise ValueError(f"N1与N2不匹配,请检查: n1 = {n1}, n2 = {n2}.") @@ -468,7 +433,7 @@ def npu_fusion_attention_backward_patch(*args, **kwargs): } new_kwargs = { - "keep_prob": 1, + "keep_prob": 1, # 同上，fusion_attention_backward 内会拦截 keep_prob 为 0 的情况 "scalar_value_value": kwargs.get("scalar_value_value", 1 / (d ** 0.5)), "sparse_mode": kwargs.get("sparse_mode", 0), "prefix": kwargs.get("prefix"), diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/cmd_parser.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/cmd_parser.py index 4af92bfa1002c419d0bd84e5dfd250b712b57136..a55df65a3772c99a6b63ebff171adb710714ab90 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/cmd_parser.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/cmd_parser.py @@ -39,6 +39,8 @@ def add_api_accuracy_checker_argument(parser): help=" The ut task result out path.") parser.add_argument("-csv_path", "--result_csv_path", dest="result_csv_path", default="", type=str, required=False, help=" the exit csv for continue") + parser.add_argument('-save_error_data', dest="save_error_data", action="store_true", + help=" Save compare failed api output.", required=False) def multi_add_api_accuracy_checker_argument(parser): @@ -49,6 +51,8 @@ def multi_add_api_accuracy_checker_argument(parser): help=" The ut task result out path.") parser.add_argument("-csv_path", "--result_csv_path", dest="result_csv_path", default="", type=str, required=False, help=" the exit csv for continue") + parser.add_argument('-save_error_data', dest="save_error_data", action="store_true", + help=" Save compare failed api output.", required=False) #以下属于多线程参数 parser.add_argument("-d", "--device", dest="device_id", nargs='+', type=int, help=" set device id to run ut, must be unique and in range 0-7", diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/data_manager.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/data_manager.py index fc2680d68a5697dae165c70a276b21038f87fbe0..24f6eb717e7ebf8fabb59d397d493831011e1161 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/data_manager.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/data_manager.py @@ -188,7 +188,7 @@ class DataManager: def record_exception_skip(self, api_name, forward_or_backward, err_msg): ''' - record exception_skip infomation into self.record_exception_skip. + record exception_skip information into self.record_exception_skip. self.record_exception_skip: dict{str: dict{"forward": str/None, "backward": str/None}} string in key is api_name, string in value is err_msg ''' @@ -270,7 +270,7 @@ class DataManager: entry.backward_pass_status, overall_err_msg ] - # change row if this api has excption_skip infomation + # change row if this api has exception_skip information if api_name in self.results_exception_skip: if self.results_exception_skip[api_name][Const.FORWARD] is not None: row[1] = CompareConst.SKIP diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/config_op.json b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/config_op.json new file mode 100644 index 0000000000000000000000000000000000000000..68a47dc26c3cb770e0e3c9a2ce2ada89dcec76c6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/config_op.json @@ -0,0 +1,9 @@ +{ + "dump_json_path": "./dump.json", + "api_name": "Mint.split.1", + "extract_api_path": "Mint.split.1.json", + "propagation": "backward", + "data_mode": "random_data", + "random_seed": 1234, + "iter_times": 1 +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..38304d525069b99367377235479cbd10ebd76158 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/op_generator.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 标准库 +import argparse +import json +import os +import re +import string + +# 应用程序自定义模块 +from msprobe.core.common.file_utils import ( + FileOpen, + load_json, + save_json, + make_dir, + change_mode, +) +from msprobe.core.common.utils import ( + check_file_or_directory_path, + check_op_str_pattern_valid, + is_int, +) +from msprobe.core.common.const import Const, MonitorConst, MsgConst, FileCheckConst +from msprobe.core.common.log import logger +from msprobe.core.common.decorator import recursion_depth_decorator + +OPERATOR_TYPE = ("Functional", "Tensor", "Torch", "Mint") + +API_INFO = 2 +FOUR_SEGMENT = 4 +FIVE_SEGMENT = 5 +DATA_NAME = "data_name" +API_MAX_LENGTH = 30 +PROPAGATION_LIST = [Const.FORWARD, Const.BACKWARD] +DATAMODE_LIST = ["random_data", "real_data"] +ITER_MAX_TIMES = 1000 +FRAMEWORK = 'framework' +REAL_DATA_PATH = 'real_data_path' +EXCLUED = {FRAMEWORK, REAL_DATA_PATH} + + +class APIInfo: + def __init__(self, api_full_name, api_info_dict, backward_info=None): + self.api_full_name = api_full_name + self.api_info_dict = api_info_dict + self.backward_info = backward_info + + @property + def api_type(self): + return self.api_full_name.split(Const.SEP, -1)[0] + + @classmethod + def from_json(cls, json_content, propagation): + forward_name, forward_dict = list(json_content.items())[0] + forward_info = cls(api_full_name=forward_name, api_info_dict=forward_dict) + + if propagation == Const.BACKWARD: + backward_name, backward_dict = list(json_content.items())[1] + backward_info = cls(api_full_name=backward_name, api_info_dict=backward_dict) + forward_info.backward_info = backward_info + + if not forward_info.is_supported_type(): + raise ValueError(f"type {forward_info.api_type} of API is not supported!") + + return forward_info + + def is_supported_type(self): + return self.api_type in OPERATOR_TYPE + + +class CommonConfig: + def __init__(self, json_config): + self.dump_json_path = json_config.get('dump_json_path') + self.api_name = json_config.get('api_name') + self.extract_api_path = json_config.get('extract_api_path') + self.propagation = json_config.get('propagation') + self.data_mode = json_config.get('data_mode') + self.random_seed = json_config.get('random_seed') + self.iter_times = json_config.get('iter_times') + self._check_config() + + def check_user_settings(self): + iter_t = self.iter_times + if iter_t <= 0 or iter_t > ITER_MAX_TIMES: + raise ValueError(f"iter_times should be range from 1 to {ITER_MAX_TIMES}.") + + json_file = self.extract_api_path + propagation = self.propagation + + json_content = load_json(json_file) + + # ensure the dict is not empty + if not json_content: + raise ValueError(f'json file is empty!') + + # ensure json_content is of type dict + if not isinstance(json_content, dict): + raise ValueError(f'content of json file is not a dict!') + + # ensure the length of json_content is within allowed limits + + filtered = {k: v for k, v in json_content.items() if k not in EXCLUED} + + if len(filtered) > API_INFO: + raise ValueError(f'json file has more than one API, the API only contains forward and backward info') + + # Retrieve the first API name and dictionary + forward_item = next(iter(json_content.items()), None) + if not forward_item or not isinstance(forward_item[1], dict) or not forward_item[1]: + raise ValueError(f'Invalid forward API data in json_content!') + + # if propagation is backward, ensure json file contains forward and backward info + if propagation == Const.BACKWARD and len(filtered) < API_INFO: + raise ValueError(f'Backward propagation requires contains forward and backward info!') + + # if propagation is backward, ensure it has valid data + if propagation == Const.BACKWARD: + backward_item = list(json_content.items())[1] + if not isinstance(backward_item[1], dict) or not backward_item[1]: + raise ValueError(f'Invalid backward API data in json_content!') + + return json_content + + def _check_config(self): + if self.dump_json_path: + check_file_or_directory_path(self.dump_json_path) + if self.api_name: + check_op_str_pattern_valid(self.api_name) + if len(self.api_name) > API_MAX_LENGTH: + raise ValueError(f'API name {self.api_name} is too long!') + make_dir(os.path.dirname(self.extract_api_path)) + if self.propagation and self.propagation not in PROPAGATION_LIST: + raise ValueError(f'propagation is invalid, it should be one of {PROPAGATION_LIST}') + if self.data_mode and self.data_mode not in DATAMODE_LIST: + raise ValueError(f'data_mode is invalid, it should be one of {DATAMODE_LIST}') + if not is_int(self.random_seed): + raise ValueError(f'random_seed is invalid, it should be an int') + if not is_int(self.iter_times): + raise ValueError(f'iter_times is invalid, it should be an int') + + +class APIExtractor: + def __init__(self, api_name, dump_json_path, output_file): + self.api_name = api_name + self.dump_json_path = dump_json_path + self.output_file = output_file + self.data = None + self.framework = None + self.real_data_path = None + + def extract_op(self): + self.data = load_json(self.dump_json_path) + # 拿到 framework + self.framework = self.data.get(FRAMEWORK, None) + + new_data = {} + extract_key_pattern = re.compile(f"^{re.escape(self.api_name)}\..+") # 修改为只要包含或等于apiname即可，不需要是只包含 + + self.real_data_path = self.data.get('dump_data_dir', '') + + for key, value in self.data.get('data', {}).items(): + if extract_key_pattern.match(key): + if self.real_data_path: + value = self.load_real_data_path(value, self.real_data_path) + new_data[key] = value + + if self.real_data_path is not None: + new_data[REAL_DATA_PATH] = self.real_data_path + + # 把 framework 加进去 + if self.framework is not None: + new_data[FRAMEWORK] = self.framework + if not new_data: + logger.warning(f"Warning: The api '{self.api_name}' does not exist in the file.") + else: + save_json(self.output_file, new_data, indent=4) + logger.info( + f"The api '{self.api_name}' has been successfully extracted and saved in: {self.output_file}") + + def load_real_data_path(self, value, dump_data_dir): + parameters = [Const.INPUT_ARGS, Const.GRAD_INPUT, Const.INPUT, Const.OUTPUT, Const.GRAD_OUTPUT] + for parameter in parameters: + for v in value.get(parameter, []): + if v is not None: + self.update_data_name(v, dump_data_dir) + return value + + @recursion_depth_decorator("OpGenerator: APIExtractor.update_data_name") + def update_data_name(self, data, dump_data_dir): + if isinstance(data, list): + for item in data: + self.update_data_name(item, dump_data_dir) + elif DATA_NAME in data: + data[DATA_NAME] = os.path.join(dump_data_dir, data[DATA_NAME]) + + +class OperatorScriptGenerator: + def __init__(self, common_config, args_info_forward, kwargs_info_forward, args_info_backward): + self.common_config = common_config + self.args_info_forward = args_info_forward + self.kwargs_info_forward = kwargs_info_forward + self.args_info_backward = args_info_backward + + @staticmethod + def extract_detailed_api_segments(full_api_name): + """ + Function Description: + Extract the name of the API. + Parameter: + full_api_name_with_direction_status: Full name of the API. Example: torch.matmul.0.forward.output.0 + Return: + api_name: Name of api. Example: matmul, mul, etc. + full_api_name: Full name of api. Example: torch.matmul.0 + direction_status: Direction status of api. Example: forward, backward, etc. + """ + api_parts = full_api_name.split(Const.SEP) + api_parts_length = len(api_parts) + api_type, api_name, api_order = None, None, None + if api_parts_length == FOUR_SEGMENT: + api_type, api_name, api_order, _ = api_parts + elif api_parts_length == FIVE_SEGMENT: + api_type, prefix, api_name, api_order, _ = api_parts + api_name = Const.SEP.join([prefix, api_name]) + return api_type, api_name, api_order + + @staticmethod + def generate_forward_inputs_code(args_info): + names = [] + + def collect(info): + if isinstance(info, dict): + names.append(info["parameter_name"]) + else: + for sub in info: + collect(sub) + + collect(args_info) + + return ( + " forward_inputs = [\n" + " ComputeElement(parameter=info)\n" + " for info in (" + ", ".join(names) + ")\n" + " ]\n" + ) + + @staticmethod + def generate_kwargs_compute_element_dict_code(): + return ( + " # ---- 构造 kwargs 对应的 ComputeElement 字典 ----\n" + " kwargs_compute_element_dict = {\n" + " key_str: ComputeElement(compute_element_info=compute_element_info)\n" + " for key_str, compute_element_info in kwargs_device.items()\n" + " }\n" + ) + + @staticmethod + def generate_gradient_inputs_code(args_info_backward): + names = [] + + def collect(info): + if isinstance(info, dict): + names.append(info["parameter_name"]) + else: + for sub in info: + collect(sub) + + collect(args_info_backward) + + return ( + " # —— 构造反向梯度 ComputeElement 列表 —— #\n" + " gradient_inputs = [\n" + " ComputeElement(parameter=info)\n" + " for info in (" + ", ".join(names) + ")\n" + " ]\n" + ) + + def get_settings(self, api_full_name): + ''' + internal_settings contain all information needed for the operator program. + keys: + api_full_name: api_type.api_name.ordinal_number + api_type: type of API, one of torch.nn.functional, torch.Tensor or Torch + api_name: name of API + ordinal_number: how many times the same api has been called + direction_status: forward + random_seed: if mode is random_data, random seed is random_seed + iter_times: if mode is random_data, generate iter_times group of data; if mode is real_data, + iter_times does not matter + args_element_assignment: code for args assignment + args_list_generator_device: code for generate args list on device + args_list_generator_bench: code for generate args list on bench + kwargs_value_assignment: code for kwargs assignment + kwargs_dict_generator_device: code for generate kwargs dict on device + kwargs_dict_generator_bench: code for generate kwargs dict on bench + ''' + # Generate an internal setting dictionary based on user settings + # including API name, type, comparison standard, random seed, number of iterations and other information + internal_settings = {} + internal_settings["propagation"] = self.common_config.propagation + internal_settings["api_full_name"] = api_full_name + api_type, api_name, ordinal_number = self.extract_detailed_api_segments(api_full_name) + if api_type == "Functional": + internal_settings["api_type"] = "torch.nn.functional" + elif api_type == "Tensor": + internal_settings["api_type"] = "torch.Tensor" + else: + internal_settings["api_type"] = "torch" + internal_settings["api_name"] = api_name + internal_settings["ordinal_number"] = ordinal_number + internal_settings["direction_status"] = self.common_config.propagation + internal_settings["random_seed"] = self.common_config.random_seed + internal_settings["data_mode"] = self.common_config.data_mode + if self.common_config.data_mode == "real_data": + internal_settings["iter_times"] = 1 + else: + internal_settings["iter_times"] = self.common_config.iter_times + + internal_settings["args_info_forward"] = self.args_info_forward + internal_settings["kwargs_info_forward"] = self.kwargs_info_forward + internal_settings["args_info_backward"] = self.args_info_backward + + return internal_settings + + +def _op_generator_parser(parser): + parser.add_argument("-i", "--config_input", dest="config_input", type=str, + help=" Path of config json file", required=True) + parser.add_argument("-o", "--api_output_path", dest="api_output_path", type=str, + help=" Path of extract api_name.json.", required=True) + + +def parse_json_config(json_file_path): + if not json_file_path: + raise Exception("config_input path can not be empty, please check.") + json_config = load_json(json_file_path) + common_config = CommonConfig(json_config) + return common_config + + +def _run_operator_generate_commond(cmd_args): + common_config = parse_json_config(cmd_args.config_input) + + if common_config.dump_json_path: + api_extract = APIExtractor(common_config.api_name, common_config.dump_json_path, common_config.extract_api_path) + api_extract.extract_op() + framework = api_extract.framework + real_data_path = api_extract.real_data_path + check_file_or_directory_path(common_config.extract_api_path) + check_file_or_directory_path(cmd_args.api_output_path, isdir=True) + json_content = common_config.check_user_settings() + api_info = APIInfo.from_json(json_content, common_config.propagation) + + if common_config.propagation == Const.BACKWARD: + # read and check json + api_full_name_forward, api_info_dict_forward = api_info.api_full_name, api_info.api_info_dict + api_full_name_backward, api_info_dict_backward = (api_info.backward_info.api_full_name, + api_info.backward_info.api_info_dict) + args_info_forward = api_info_dict_forward.get(Const.INPUT_ARGS) + kwargs_info_forward = api_info_dict_forward.get(Const.INPUT_KWARGS) + if Const.GRAD_INPUT in api_info_dict_backward: + args_info_backward = api_info_dict_backward.get(Const.GRAD_INPUT) + elif Const.INPUT in api_info_dict_backward: + args_info_backward = api_info_dict_backward.get(Const.INPUT) + op_generate = OperatorScriptGenerator(common_config, args_info_forward, kwargs_info_forward, args_info_backward) + internal_settings = op_generate.get_settings(api_full_name_backward) + internal_settings[FRAMEWORK] = framework + internal_settings[REAL_DATA_PATH] = real_data_path + else: + # read and check json + api_full_name_forward, api_info_dict_forward = api_info.api_full_name, api_info.api_info_dict + + args_info_forward = api_info_dict_forward.get(Const.INPUT_ARGS) + + kwargs_info_forward = api_info_dict_forward.get(Const.INPUT_KWARGS) + + op_generate = OperatorScriptGenerator(common_config, args_info_forward, kwargs_info_forward, None) + internal_settings = op_generate.get_settings(api_full_name_forward) + internal_settings[FRAMEWORK] = framework + internal_settings[REAL_DATA_PATH] = real_data_path + + template_path = os.path.join(os.path.dirname(__file__), "operator_replication.template") + operator_script_path = os.path.join(cmd_args.api_output_path, + "{0}.py".format(internal_settings.get("api_full_name"))) + + class SafeDict(dict): + def __missing__(self, key): + # leave {key} in the output if it’s not in the dict + return '{' + key + '}' + + class RobustFormatter(string.Formatter): + def vformat(self, format_string, args, kwargs): + result = [] + # parse() 会把文本和每个占位符拆开 + for literal, field_name, format_spec, conversion in self.parse(format_string): + # 输出字面文本 + result.append(literal) + if field_name is None: + continue + try: + # 正常获取变量并格式化 + obj, _ = self.get_field(field_name, args, kwargs) + if conversion: + obj = self.convert_field(obj, conversion) + result.append(self.format_field(obj, format_spec)) + except Exception: + # 不管是 KeyError 还是 ValueError，都原样回写 {field_name[:format_spec]} + placeholder = '{' + field_name + if conversion: + placeholder += '!' + conversion + if format_spec: + placeholder += ':' + format_spec + placeholder += '}' + result.append(placeholder) + return ''.join(result) + + fmt = RobustFormatter() + with FileOpen(template_path, 'r') as ftemp, FileOpen(operator_script_path, 'w') as fout: + code_template = ftemp.read() + # 这里用 fmt.format，不用 format_map + fout.write(fmt.format(code_template, **internal_settings)) + + change_mode(operator_script_path, FileCheckConst.DATA_FILE_AUTHORITY) + + logger.info(f"Generate operator script successfully and the name is {operator_script_path}.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + _op_generator_parser(parser) + cmd_args = parser.parse_args() + _run_operator_generate_commond(cmd_args) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/operator_replication.template b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/operator_replication.template new file mode 100644 index 0000000000000000000000000000000000000000..b4e5747112218a6d89b6b1255b57ebcfed07c5fe --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/generate_op_script/operator_replication.template @@ -0,0 +1,2070 @@ +import os +import re +import stat +import time +from enum import Enum, auto +from abc import ABC, abstractmethod +import csv + +import gc +import sys +from pathlib import Path +import mindspore +from mindspore import ops + + +from tabulate import tabulate + +import logging + +import traceback + + + +def error_log_with_exp(self, msg: str, exp: Exception): + """ + msg: 你的错误提示 + exp: 你要记录的 Exception 实例 + """ + # 将 Exception 的类型、消息和 traceback 通过 exc_info 参数一并传给 .error() + self.error(msg, exc_info=(type(exp), exp, exp.__traceback__)) + +# 把它挂到 Logger 上 +logging.Logger.error_log_with_exp = error_log_with_exp + + + +# 1. 基本配置：设置日志级别为 INFO，默认输出到控制台 +logging.basicConfig(level=logging.INFO, + format='%(asctime)s [%(levelname)s] %(message)s', + datefmt='%H:%M:%S') + +logger = logging.getLogger() + + +# ======= 常数类 ======= + +class CodedException(Exception): + def __init__(self, code, error_info=''): + super().__init__() + self.code = code + self.error_info = self.err_strs.get(code) + error_info + + def __str__(self): + return self.error_info + + +class ApiAccuracyCheckerException(CodedException): + ParseJsonFailed = 0 + UnsupportType = 1 + WrongValue = 2 + ApiWrong = 3 + err_strs = { + ParseJsonFailed: "[msprobe] Api Accuracy Checker parse json failed: ", + UnsupportType: "[msprobe] Api Accuracy Checker get unsupported type: ", + WrongValue: "[msprobe] Api Accuracy Checker get wrong value: ", + ApiWrong: "[msprobe] Api Accuracy Checker something wrong with api: ", + } + + +class FileCheckConst: + """ + Class for file check const + """ + READ_ABLE = "read" + WRITE_ABLE = "write" + READ_WRITE_ABLE = "read and write" + DIRECTORY_LENGTH = 4096 + FILE_NAME_LENGTH = 255 + FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" + FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$' + PKL_SUFFIX = ".pkl" + NUMPY_SUFFIX = ".npy" + JSON_SUFFIX = ".json" + PT_SUFFIX = ".pt" + CSV_SUFFIX = ".csv" + XLSX_SUFFIX = ".xlsx" + YAML_SUFFIX = ".yaml" + IR_SUFFIX = ".ir" + ZIP_SUFFIX = ".zip" + SHELL_SUFFIX = ".sh" + MAX_PKL_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 + MAX_NUMPY_SIZE = 10737418240 # 10 * 1024 * 1024 * 1024 + MAX_JSON_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 + MAX_PT_SIZE = 10737418240 # 10 * 1024 * 1024 * 1024 + MAX_CSV_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 + MAX_XLSX_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 + MAX_YAML_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 + MAX_IR_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 + MAX_ZIP_SIZE = 10737418240 # 10 * 1024 * 1024 * 1024 + MAX_FILE_IN_ZIP_SIZE = 1073741824 # 1 * 1024 * 1024 * 1024 + COMMOM_FILE_SIZE = 1048576 # 1 * 1024 * 1024 + DIR = "dir" + FILE = "file" + DATA_DIR_AUTHORITY = 0o750 + DATA_FILE_AUTHORITY = 0o640 + FILE_SIZE_DICT = { + PKL_SUFFIX: MAX_PKL_SIZE, + NUMPY_SUFFIX: MAX_NUMPY_SIZE, + JSON_SUFFIX: MAX_JSON_SIZE, + PT_SUFFIX: MAX_PT_SIZE, + CSV_SUFFIX: MAX_CSV_SIZE, + XLSX_SUFFIX: MAX_XLSX_SIZE, + YAML_SUFFIX: MAX_YAML_SIZE, + IR_SUFFIX: MAX_IR_SIZE, + ZIP_SUFFIX: MAX_ZIP_SIZE + } + CSV_BLACK_LIST = r'^[＋－＝％＠\+\-=%@]|;[＋－＝％＠\+\-=%@]' + +class Const: + MAX_DEPTH = 10 + PT_FRAMEWORK = "pytorch" + MS_FRAMEWORK = "mindspore" + MT_FRAMEWORK = "mindtorch" + SEP = "." + KWARGS = 'kwargs' + INPUT = 'input' + OUTPUT = 'output' + INPUT_ARGS = 'input_args' + INPUT_KWARGS = 'input_kwargs' + GRAD_INPUT = 'grad_input' + GRAD_OUTPUT = 'grad_output' + BACKWARD = 'backward' + FORWARD = 'forward' + + +class CompareConst: + # compare result data + PASS = 'pass' + WARNING = 'Warning' + ERROR = 'error' + TRUE = 'TRUE' + FALSE = 'FALSE' + SKIP = 'SKIP' + + # compare result column name + COSINE = "Cosine" + EUC_DIST = "EucDist" + MAX_ABS_ERR = "MaxAbsErr" + MAX_RELATIVE_ERR = "MaxRelativeErr" + MIN_RELATIVE_ERR = "MinRelativeErr" + MEAN_RELATIVE_ERR = "MeanRelativeErr" + NORM_RELATIVE_ERR = "NormRelativeErr" + + # accuracy standards + COS_THRESHOLD = 0.99 + MAX_ABS_ERR_THRESHOLD = 0.001 + MAX_RELATIVE_ERR_THRESHOLD = 0.001 + COS_MAX_THRESHOLD = 0.9 + MAX_ABS_ERR_MAX_THRESHOLD = 1 + +class MsCompareConst: + # api_info field + MINT = "Mint" + MINT_FUNCTIONAL = "MintFunctional" + TENSOR_API = "Tensor" + FUNCTIONAL_API = "Functional" + FUSION_API = "FUSION" + + API_NAME_STR_LENGTH = 4 + MAX_RECURSION_DEPTH = 20 + + # Mindtorch api_info field + MINDTORCH_TENSOR = "Tensor" + MINDTORCH = "Torch" + MINDTORCH_FUNC = "Functional" + MINDTORCH_NPU = "NPU" + MINDTORCH_DIST = "Distributed" + + MT_VALID_API_TYPES = [ + MINDTORCH, MINDTORCH_FUNC, MINDTORCH_TENSOR + ] + SUPPORTED_FUSION_LIST = ["flash_attention_score"] + + TASK_FIELD = "task" + STATISTICS_TASK = "statistics" + FRAMEWORK = "framework" + TENSOR_TASK = "tensor" + DUMP_DATA_DIR_FIELD = "dump_data_dir" + DATA_FIELD = "data" + + # supported api yaml + SUPPORTED_API_LIST_FILE = "checker_support_api.yaml" + SUPPORTED_TENSOR_LIST_KEY = "tensor" + + # detail_csv + DETAIL_CSV_API_NAME = "API Name" + DETAIL_CSV_BENCH_DTYPE = "Bench Dtype" + DETAIL_CSV_TESTED_DTYPE = "Tested Dtype" + DETAIL_CSV_SHAPE = "Shape" + DETAIL_CSV_PASS_STATUS = "Status" + DETAIL_CSV_MESSAGE = "Message" + DETAIL_CSV_FILE_NAME = "accuracy_checking_details" + + # result_csv + RESULT_CSV_FORWARD_TEST_SUCCESS = "Forward Test Success" + RESULT_CSV_BACKWARD_TEST_SUCCESS = "Backward Test Success" + RESULT_CSV_FILE_NAME = "accuracy_checking_result" + + EPSILON = 1e-8 + + class ProcessStatus: + SUCCESS = "success" + API_NOT_FOUND = "api_not_found" + EXCEPTION_SKIP = "exception_skip" + +# ======= mindtorch支持 ======== + +import torch as mindtorch +from torch import Tensor as mindtorch_tensor +import torch.nn.functional as mindtorch_func +import torch.distributed as mindtorch_dist + +is_valid_pt_mt_env = True + + +def is_mindtorch(): + mindtorch_check_result = False + try: + import torch as test_torch + from mindspore import Tensor as MindsporeTensor + except ImportError: + return mindtorch_check_result + tensor = test_torch.tensor(0.0) + if isinstance(tensor, MindsporeTensor): + mindtorch_check_result = True + + return mindtorch_check_result + + +def remove_torch_related_paths(): + removed_paths = [] + if not is_mindtorch(): + return + try: + import torch as remove_torch + torch_file = remove_torch.__file__ + except ImportError: + return + + torch_dir = os.path.dirname(torch_file) + + torch_dir_path = Path(torch_dir).resolve() + parent_dir = torch_dir_path.parent + + paths_to_remove = [str(parent_dir)] + + for path in paths_to_remove: + try: + path_resolved = str(Path(path).resolve()) + except Exception as error: + logger.debug(f"Failed to resolve path {path}: {error}") + + + if path_resolved in sys.path: + index = sys.path.index(path_resolved) + removed_paths.append((path_resolved, index)) + sys.path.pop(index) + + return + + +def clear_torch_from_sys_modules(): + modules_to_remove = [] + for module in sys.modules: + if module == "torch" or module.startswith("torch."): + modules_to_remove.append(module) + + for module in modules_to_remove: + del sys.modules[module] + + +def set_pt_mt_env_invalid(): + global is_valid_pt_mt_env + is_valid_pt_mt_env = False + + +def delete_torch_paths(): + + if not is_mindtorch(): + set_pt_mt_env_invalid() + + clear_torch_from_sys_modules() + + for count_delete_env_path in range(MsCompareConst.MAX_RECURSION_DEPTH): + if not is_mindtorch(): + break + + remove_torch_related_paths() + + clear_torch_from_sys_modules() + + if count_delete_env_path >= MsCompareConst.MAX_RECURSION_DEPTH - 1: + raise Exception(f"Please check if you have a valid PyTorch and MindTorch environment, and ensure " + f"the PYTHONPATH environment variable depth does not exceed {Const.MAX_RECURSION_DEPTH}.") + + +if not is_mindtorch(): + set_pt_mt_env_invalid() + +else: + initial_sys_path = sys.path.copy() + delete_torch_paths() + + gc.collect() + + import torch + + if is_mindtorch(): + set_pt_mt_env_invalid() + + sys.path = initial_sys_path + + + +if not is_valid_pt_mt_env: + import torch + + + +# ======= 常数类 ======= + +import numpy as np +from mindspore._c_expression import typing +from mindspore.common import dtype as mstype + + +TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"] +TORCH_BOOL_TYPE = ["torch.bool"] +TORCH_INT_TYPE = ["torch.uint8", "torch.int8", "torch.int16", "torch.short", "torch.int32", "torch.int", + "torch.int64", "torch.long"] +TORCH_FLOAT_TYPE = ["torch.float16", "torch.half", "torch.bfloat16", "torch.float32", "torch.float", + "torch.float64", "torch.double"] +TORCH_COMPLEX_TYPE = ["torch.complex32", "torch.chalf", "torch.complex64", "torch.cfloat", "torch.complex128", "torch.cdouble"] +RAISE_PRECISION = {{ + "torch.float16": torch.float32, + "torch.half": torch.float32, + "torch.bfloat16": torch.float32, + "torch.float32": torch.float64, + "torch.float": torch.float64 +}} +THOUSANDTH_THRESHOLDING = 0.001 +BACKWARD = 'backward' +DIR = "dir" +FILE = "file" +READ_ABLE = "read" +WRITE_ABLE = "write" +READ_WRITE_ABLE = "read and write" +DIRECTORY_LENGTH = 4096 +FILE_NAME_LENGTH = 255 +SOFT_LINK_ERROR = "检测到软链接" +FILE_PERMISSION_ERROR = "文件权限错误" +INVALID_FILE_ERROR = "无效文件" +ILLEGAL_PATH_ERROR = "非法文件路径" +ILLEGAL_PARAM_ERROR = "非法打开方式" +FILE_TOO_LARGE_ERROR = "文件过大" +FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" +FILE_SIZE_DICT = {{ + ".pkl": 1073741824, # 1 * 1024 * 1024 * 1024 + ".npy": 10737418240, # 10 * 1024 * 1024 * 1024 + ".json": 1073741824, # 1 * 1024 * 1024 * 1024 + ".pt": 10737418240, # 10 * 1024 * 1024 * 1024 + ".csv": 1073741824, # 1 * 1024 * 1024 * 1024 + ".xlsx": 1073741824, # 1 * 1024 * 1024 * 1024 + ".yaml": 1073741824, # 1 * 1024 * 1024 * 1024 + ".ir": 1073741824 # 1 * 1024 * 1024 * 1024 +}} +COMMOM_FILE_SIZE = 1048576 # 1 * 1024 * 1024 + + +INT8 = "Int8" +UINT8 = "UInt8" +INT16 = "Int16" +UINT16 = "UInt16" +INT32 = "Int32" +UINT32 = "UInt32" +INT64 = "Int64" +UINT64 = "UInt64" +FLOAT16 = "Float16" +FLOAT32 = "Float32" +FLOAT64 = "Float64" +BOOL = "Bool" +BFLOAT16 = "BFloat16" +INT4 = "Int4" + +dtype_str_to_ms_dtype = { + INT8: mstype.int8, + UINT8: mstype.uint8, + INT16: mstype.int16, + UINT16: mstype.uint16, + INT32: mstype.int32, + UINT32: mstype.uint32, + INT64: mstype.int64, + UINT64: mstype.uint64, + FLOAT16: mstype.float16, + FLOAT32: mstype.float32, + FLOAT64: mstype.float64, + BOOL: mstype.bool_, + BFLOAT16: mstype.bfloat16, + INT4: mstype.qint4x2 +} +ms_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_ms_dtype.items()} + +dtype_str_to_np_dtype = { + INT8: np.int8, + UINT8: np.uint8, + INT16: np.int16, + UINT16: np.uint16, + INT32: np.int32, + UINT32: np.uint32, + INT64: np.int64, + UINT64: np.uint64, + FLOAT16: np.float16, + FLOAT32: np.float32, + FLOAT64: np.float64, + BOOL: np.bool_ +} +np_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_np_dtype.items()} + +dtype_str_to_torch_dtype = { + INT8: torch.int8, + UINT8: torch.uint8, + INT16: torch.int16, + INT32: torch.int32, + INT64: torch.int64, + FLOAT16: torch.float16, + FLOAT32: torch.float32, + FLOAT64: torch.float64, + BOOL: torch.bool, + BFLOAT16: torch.bfloat16, +} +torch_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_torch_dtype.items()} + + +dtype_str_to_mindtorch_dtype = { + INT8: mindtorch.int8, + UINT8: mindtorch.uint8, + INT16: mindtorch.int16, + INT32: mindtorch.int32, + INT64: mindtorch.int64, + FLOAT16: mindtorch.float16, + FLOAT32: mindtorch.float32, + FLOAT64: mindtorch.float64, + BOOL: mindtorch.bool, + BFLOAT16: mindtorch.bfloat16, +} +mindtorch_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_mindtorch_dtype.items()} + +MINDSPORE_TENSOR_TYPE_STR = "mindspore.Tensor" +BOOL_TYPE_STR = "bool" +INT_TYPE_STR = "int" +FLOAT_TYPE_STR = "float" +SLICE_TYPE_STR = "slice" +TUPLE_TYPE_STR = "tuple" +STR_TYPE_STR = "str" +MINDSPORE_DTYPE_TYPE_STR = "mindspore.dtype" +TORCH_DTYPE_TYPE_STR = "torch.dtype" + +api_info_type_str_to_type = { + MINDSPORE_TENSOR_TYPE_STR: mindspore.Tensor, + BOOL_TYPE_STR: bool, + INT_TYPE_STR: int, + FLOAT_TYPE_STR: float, + SLICE_TYPE_STR: slice, + STR_TYPE_STR: str, + MINDSPORE_DTYPE_TYPE_STR: typing.Type, +} +type_to_api_info_type_str = {value: key for key, value in api_info_type_str_to_type.items()} + +DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE = np.float64 +DEFAULT_CONSTRUCT_NP_INT_DTYPE = np.float64 +DEFAULT_CONSTRUCT_NP_UINT_DTYPE = np.float64 + +float_dtype_str_list = [ + FLOAT16, + FLOAT32, + FLOAT64, + BFLOAT16, +] + +int_dtype_str_list = [ + INT8, + INT16, + INT32, + INT64, + BOOL, + INT4, +] + +uint_dtype_str_list = [ + UINT8, + UINT16, + UINT32, + UINT64, +] + +# ======= 比对类 ======= + +class CompareResult: + def __init__(self, compare_value, pass_status, err_msg): + self.compare_value = compare_value + self.pass_status = pass_status + self.err_msg = err_msg + + +class BaseCompareAlgorithm(ABC): + def __init__(self) -> None: + super().__init__() + self.compare_algorithm_name = None + self.err_msg_mapping = { + CompareConst.COSINE: { + CompareConst.PASS: "", + CompareConst.ERROR: f"cosine similarity is less than threshold: {CompareConst.COS_THRESHOLD} ", + CompareConst.SKIP: "two inputs are not valid for computing cosine similarity, skip comparing ", + }, + CompareConst.MAX_ABS_ERR: { + CompareConst.PASS: "", + CompareConst.ERROR: "max absolute difference is greater than " \ + f"threshold: {CompareConst.MAX_ABS_ERR_THRESHOLD} ", + CompareConst.SKIP: "two inputs are not valid for computing max absolute difference, skip comparing ", + }, + CompareConst.MAX_RELATIVE_ERR: { + CompareConst.PASS: "", + CompareConst.ERROR: "", + CompareConst.SKIP: "", + }, + } + + def __call__(self, bench_compute_element, tested_compute_element): + ''' + Args: + bench_compute_element: ComputeElement + tested_compute_element: ComputeElement + + Return: + compare_result: CompareResult + ''' + if self.check_validity(bench_compute_element, tested_compute_element): + compare_value = self.run_compare(bench_compute_element, tested_compute_element) + pass_status = self.check_pass(compare_value) + else: + logger.warning(f"not suitable for computing {self.compare_algorithm_name}, skip this.") + compare_value = None + pass_status = CompareConst.SKIP + + err_msg = self.err_msg_mapping.get(self.compare_algorithm_name).get(pass_status) + + compare_result = CompareResult(compare_value, pass_status, err_msg) + return compare_result + + @staticmethod + def convert_to_np_float64_ndarray(tensor): + if isinstance(tensor, mindspore.Tensor): + ndarray = tensor.astype(mindspore.float64).numpy() + elif isinstance(tensor, torch.Tensor): + ndarray = tensor.to(torch.float64, copy=True).numpy() + else: + err_msg = "BaseCompareAlgorithm.convert_to_np_float64_ndarray failed: " \ + "input is not mindspore.Tensor or torch.Tensor" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + return ndarray + + @staticmethod + def check_two_tensor(bench_compute_element, tested_compute_element): + bench_parameter = bench_compute_element.get_parameter() + tested_parameter = tested_compute_element.get_parameter() + + bench_is_tensor = isinstance(bench_parameter, (mindspore.Tensor, torch.Tensor)) + tested_is_tensor = isinstance(tested_parameter, (mindspore.Tensor, torch.Tensor)) + shape_same = bench_compute_element.get_shape() == tested_compute_element.get_shape() + return bench_is_tensor and tested_is_tensor and shape_same + + @abstractmethod + def check_validity(self, bench_compute_element, tested_compute_element): + ''' + Args: + bench_compute_element: ComputeElement + tested_compute_element: ComputeElement + + Return: + check_res: boolean + ''' + raise NotImplementedError + + @abstractmethod + def run_compare(self, bench_compute_element, tested_compute_element): + ''' + Args: + bench_compute_element: ComputeElement + tested_compute_element: ComputeElement + + Return: + compare_value: float/int + ''' + raise NotImplementedError + + @abstractmethod + def check_pass(self, compare_value): + ''' + Args: + compare_value: float/int + + Return: + pass_status: str + ''' + raise NotImplementedError + + +class CosineSimilarityCompareAlgorithm(BaseCompareAlgorithm): + def __init__(self) -> None: + super().__init__() + self.compare_algorithm_name = CompareConst.COSINE + + def check_validity(self, bench_compute_element, tested_compute_element): + return self.check_two_tensor(bench_compute_element, tested_compute_element) + + def run_compare(self, bench_compute_element, tested_compute_element): + bench_ndarray = self.convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + + bench_norm = np.linalg.norm(bench_ndarray) + tested_norm = np.linalg.norm(tested_ndarray) + dot_product = np.dot(bench_ndarray.flatten(), tested_ndarray.flatten()) + cosine_similarity = (MsCompareConst.EPSILON + dot_product) / (MsCompareConst.EPSILON + bench_norm * tested_norm) + return cosine_similarity + + def check_pass(self, compare_value): + if compare_value > CompareConst.COS_THRESHOLD: + return CompareConst.PASS + else: + return CompareConst.ERROR + + +class MaxAbsoluteDiffCompareAlgorithm(BaseCompareAlgorithm): + def __init__(self) -> None: + super().__init__() + self.compare_algorithm_name = CompareConst.MAX_ABS_ERR + + def check_validity(self, bench_compute_element, tested_compute_element): + return self.check_two_tensor(bench_compute_element, tested_compute_element) + + def run_compare(self, bench_compute_element, tested_compute_element): + bench_ndarray = self.convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + + max_absolute_diff = np.max(np.abs(bench_ndarray - tested_ndarray)) + return max_absolute_diff + + def check_pass(self, compare_value): + if compare_value < CompareConst.MAX_ABS_ERR_THRESHOLD: + return CompareConst.PASS + else: + return CompareConst.ERROR + + +class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): + def __init__(self) -> None: + super().__init__() + self.compare_algorithm_name = CompareConst.MAX_RELATIVE_ERR + + def check_validity(self, bench_compute_element, tested_compute_element): + return self.check_two_tensor(bench_compute_element, tested_compute_element) + + def run_compare(self, bench_compute_element, tested_compute_element): + bench_ndarray = self.convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + + abs_diff = np.abs(bench_ndarray - tested_ndarray) + bench_ndarray_nonzero = np.abs(bench_ndarray) + (bench_ndarray == 0) * MsCompareConst.EPSILON + max_relative_diff = np.max(abs_diff / bench_ndarray_nonzero) + return max_relative_diff + + def check_pass(self, compare_value): + if compare_value < CompareConst.MAX_RELATIVE_ERR_THRESHOLD: + return CompareConst.PASS + else: + return CompareConst.ERROR + + +compare_algorithms = { + CompareConst.COSINE: CosineSimilarityCompareAlgorithm(), + CompareConst.MAX_ABS_ERR: MaxAbsoluteDiffCompareAlgorithm(), + CompareConst.MAX_RELATIVE_ERR: MaxRelativeDiffCompareAlgorithm(), +} + + + +class CompareStandard(Enum): + BINARY_EQUALITY_STANDARD = auto() + ABSOLUTE_THRESHOLD_STANDARD = auto() + ULP_ERROR_STANDARD = auto() + BENCHMARK_STANDARD = auto() + THOUSANDTH_STANDARD = auto() + + +class CompareStandard(Enum): + BINARY_EQUALITY_STANDARD = auto() + ABSOLUTE_THRESHOLD_STANDARD = auto() + ULP_ERROR_STANDARD = auto() + BENCHMARK_STANDARD = auto() + THOUSANDTH_STANDARD = auto() + + +# ======== 文件操作类 ========== + +from collections import defaultdict +from functools import wraps + + +def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_type=None, accepted_value=None): + ''' + Args: + dict_instance: dict, dict parsed from input json + key: str + key_description: str + accepted_type: tuple + accepted_value: Union[tuple, list] + + Return: + value, the corresponding value of "key" in "dict_instance" + + Exception: + raise ApiAccuracyCheckerException.ParseJsonFailed error when + 1. dict_instance is not a dict + 2. value is None + 3. value is not accepted type + 4. value is not accepted value + ''' + if not isinstance(dict_instance, dict): + error_info = "check_and_get_from_json_dict failed: input is not a dict" + raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info) + value = dict_instance.get(key) + if value is None: + error_info = f"check_and_get_from_json_dict failed: {key_description} is missing" + raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info) + elif accepted_type is not None and not isinstance(value, accepted_type): + error_info = f"check_and_get_from_json_dict failed: {key_description} is not accepted type: {accepted_type}" + raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info) + elif accepted_value is not None and value not in accepted_value: + error_info = f"check_and_get_from_json_dict failed: {key_description} is not accepted value: {accepted_value}" + raise ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed, error_info) + return value + + +def convert_to_tuple(args): + if isinstance(args, (tuple, list)): + return tuple(args) + else: + input_list = [args] + return tuple(input_list) + + +def trim_output_compute_element_list(compute_element_list, forward_or_backward): + ''' + Args: + compute_element_list: List[ComputeElement] + forward_or_backward: str, Union["forward", "backward"] + ''' + trimmed_list = [] + for compute_element in compute_element_list: + if compute_element.get_parameter() is None or \ + (forward_or_backward == Const.BACKWARD and compute_element.get_dtype() not in float_dtype_str_list): + # trim case: 1. parameter is None. 2. backward output has non float parameter + continue + trimmed_list.append(compute_element) + return trimmed_list + + + + +# 记录工具函数递归的深度 +recursion_depth = defaultdict(int) + + +def recursion_depth_decorator(func_info, max_depth=Const.MAX_DEPTH): + """装饰一个函数，当函数递归调用超过限制时，抛出异常并打印函数信息。""" + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + func_id = id(func) + recursion_depth[func_id] += 1 + + try: + result = func(*args, **kwargs) + finally: + recursion_depth[func_id] -= 1 + return result + + return wrapper + + return decorator + + + +class FileChecker: + """ + The class for check file. + + Attributes: + file_path: The file or dictionary path to be verified. + path_type: file or dictionary + ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability + file_type(str): The correct file type for file + """ + + def __init__(self, file_path, path_type, ability=None, file_type=None, is_script=True): + self.file_path = file_path + self.path_type = self._check_path_type(path_type) + self.ability = ability + self.file_type = file_type + self.is_script = is_script + + @staticmethod + def _check_path_type(path_type): + if path_type not in [FileCheckConst.DIR, FileCheckConst.FILE]: + logger.error(f'The path_type must be {FileCheckConst.DIR} or {FileCheckConst.FILE}.') + raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR) + return path_type + + def common_check(self): + """ + 功能：用户校验基本文件权限：软连接、文件长度、是否存在、读写权限、文件属组、文件特殊字符 + 注意：文件后缀的合法性，非通用操作，可使用其他独立接口实现 + """ + check_path_exists(self.file_path) + check_link(self.file_path) + self.file_path = os.path.realpath(self.file_path) + check_path_length(self.file_path) + check_path_type(self.file_path, self.path_type) + self.check_path_ability() + if self.is_script: + check_path_owner_consistent(self.file_path) + check_path_pattern_valid(self.file_path) + check_common_file_size(self.file_path) + check_file_suffix(self.file_path, self.file_type) + if self.path_type == FileCheckConst.FILE: + check_dirpath_before_read(self.file_path) + return self.file_path + + def check_path_ability(self): + if self.ability == FileCheckConst.WRITE_ABLE: + check_path_writability(self.file_path) + if self.ability == FileCheckConst.READ_ABLE: + check_path_readability(self.file_path) + if self.ability == FileCheckConst.READ_WRITE_ABLE: + check_path_readability(self.file_path) + check_path_writability(self.file_path) + + +class FileOpen: + """ + The class for open file by a safe way. + + Attributes: + file_path: The file or dictionary path to be opened. + mode(str): The file open mode + """ + SUPPORT_READ_MODE = ["r", "rb"] + SUPPORT_WRITE_MODE = ["w", "wb", "a", "ab"] + SUPPORT_READ_WRITE_MODE = ["r+", "rb+", "w+", "wb+", "a+", "ab+"] + + def __init__(self, file_path, mode, encoding='utf-8'): + self.file_path = file_path + self.mode = mode + self.encoding = encoding + self._handle = None + + def __enter__(self): + self.check_file_path() + binary_mode = "b" + if binary_mode not in self.mode: + self._handle = open(self.file_path, self.mode, encoding=self.encoding) + else: + self._handle = open(self.file_path, self.mode) + return self._handle + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._handle: + self._handle.close() + + def check_file_path(self): + support_mode = self.SUPPORT_READ_MODE + self.SUPPORT_WRITE_MODE + self.SUPPORT_READ_WRITE_MODE + if self.mode not in support_mode: + logger.error("File open not support %s mode" % self.mode) + raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR) + check_link(self.file_path) + self.file_path = os.path.realpath(self.file_path) + check_path_length(self.file_path) + self.check_ability_and_owner() + check_path_pattern_valid(self.file_path) + if os.path.exists(self.file_path): + check_common_file_size(self.file_path) + check_dirpath_before_read(self.file_path) + + def check_ability_and_owner(self): + if self.mode in self.SUPPORT_READ_MODE: + check_path_exists(self.file_path) + check_path_readability(self.file_path) + check_path_owner_consistent(self.file_path) + if self.mode in self.SUPPORT_WRITE_MODE and os.path.exists(self.file_path): + check_path_writability(self.file_path) + check_path_owner_consistent(self.file_path) + if self.mode in self.SUPPORT_READ_WRITE_MODE and os.path.exists(self.file_path): + check_path_readability(self.file_path) + check_path_writability(self.file_path) + check_path_owner_consistent(self.file_path) + + +def check_link(path): + abs_path = os.path.abspath(path) + if os.path.islink(abs_path): + logger.error('The file path {} is a soft link.'.format(path)) + raise FileCheckException(FileCheckException.SOFT_LINK_ERROR) + + +def check_path_length(path, name_length=None): + file_max_name_length = name_length if name_length else FileCheckConst.FILE_NAME_LENGTH + if len(path) > FileCheckConst.DIRECTORY_LENGTH or \ + len(os.path.basename(path)) > file_max_name_length: + logger.error('The file path length exceeds limit.') + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) + + +def check_path_exists(path): + if not os.path.exists(path): + logger.error('The file path %s does not exist.' % path) + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) + + +def check_path_readability(path): + if not os.access(path, os.R_OK): + logger.error('The file path %s is not readable.' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_path_writability(path): + if not os.access(path, os.W_OK): + logger.error('The file path %s is not writable.' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_path_executable(path): + if not os.access(path, os.X_OK): + logger.error('The file path %s is not executable.' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_other_user_writable(path): + st = os.stat(path) + if st.st_mode & 0o002: + logger.error('The file path %s may be insecure because other users have write permissions. ' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_path_owner_consistent(path): + file_owner = os.stat(path).st_uid + if file_owner != os.getuid() and os.getuid() != 0: + logger.error('The file path %s may be insecure because is does not belong to you.' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_path_pattern_valid(path): + if not re.match(FileCheckConst.FILE_VALID_PATTERN, path): + logger.error('The file path %s contains special characters.' % (path)) + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) + + +def check_file_size(file_path, max_size): + try: + file_size = os.path.getsize(file_path) + except OSError as os_error: + logger.error(f'Failed to open "{file_path}". {str(os_error)}') + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) from os_error + if file_size >= max_size: + logger.error(f'The size ({file_size}) of {file_path} exceeds ({max_size}) bytes, tools not support.') + raise FileCheckException(FileCheckException.FILE_TOO_LARGE_ERROR) + + +def check_common_file_size(file_path): + if os.path.isfile(file_path): + for suffix, max_size in FileCheckConst.FILE_SIZE_DICT.items(): + if file_path.endswith(suffix): + check_file_size(file_path, max_size) + return + check_file_size(file_path, FileCheckConst.COMMOM_FILE_SIZE) + + +def check_file_suffix(file_path, file_suffix): + if file_suffix: + if not file_path.endswith(file_suffix): + logger.error(f"The {file_path} should be a {file_suffix} file!") + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) + + +def check_path_type(file_path, file_type): + if file_type == FileCheckConst.FILE: + if not os.path.isfile(file_path): + logger.error(f"The {file_path} should be a file!") + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) + if file_type == FileCheckConst.DIR: + if not os.path.isdir(file_path): + logger.error(f"The {file_path} should be a dictionary!") + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) + +def make_dir(dir_path): + check_path_before_create(dir_path) + dir_path = os.path.realpath(dir_path) + if os.path.isdir(dir_path): + return + try: + os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) + except OSError as ex: + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, + f"Failed to create {dir_path}. " + f"Please check the path permission or disk space. {str(ex)}") from ex + file_check = FileChecker(dir_path, FileCheckConst.DIR) + file_check.common_check() + + + + +@recursion_depth_decorator('msprobe.core.common.file_utils.create_directory', max_depth=16) +def create_directory(dir_path): + """ + Function Description: + creating a safe directory with specified permissions + Parameter: + dir_path: directory path + Exception Description: + when invalid data throw exception + """ + check_link(dir_path) + check_path_before_create(dir_path) + dir_path = os.path.realpath(dir_path) + parent_dir = os.path.dirname(dir_path) + if not os.path.isdir(parent_dir): + create_directory(parent_dir) + make_dir(dir_path) + + +def check_path_before_create(path): + check_link(path) + if path_len_exceeds_limit(path): + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, 'The file path length exceeds limit.') + + if not re.match(FileCheckConst.FILE_PATTERN, os.path.realpath(path)): + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, + 'The file path {} contains special characters.'.format(path)) + + +def check_dirpath_before_read(path): + path = os.path.realpath(path) + dirpath = os.path.dirname(path) + + +def check_file_or_directory_path(path, isdir=False): + """ + Function Description: + check whether the path is valid + Parameter: + path: the path to check + isdir: the path is dir or file + Exception Description: + when invalid data throw exception + """ + if isdir: + path_checker = FileChecker(path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) + else: + path_checker = FileChecker(path, FileCheckConst.FILE, FileCheckConst.READ_ABLE) + path_checker.common_check() + + +def change_mode(path, mode): + if not os.path.exists(path) or os.path.islink(path): + return + try: + os.chmod(path, mode) + except PermissionError as ex: + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR, + 'Failed to change {} authority. {}'.format(path, str(ex))) from ex + + +def path_len_exceeds_limit(file_path): + return len(os.path.realpath(file_path)) > FileCheckConst.DIRECTORY_LENGTH or \ + len(os.path.basename(file_path)) > FileCheckConst.FILE_NAME_LENGTH + +def load_npy(filepath): + check_file_or_directory_path(filepath) + try: + npy = np.load(filepath, allow_pickle=False) + except Exception as e: + logger.error(f"The numpy file failed to load. Please check the path: {filepath}.") + raise RuntimeError(f"Load numpy file {filepath} failed.") from e + return npy + +def write_csv(data, filepath, mode="a+", malicious_check=False): + def csv_value_is_valid(value: str) -> bool: + if not isinstance(value, str): + return True + try: + # -1.00 or +1.00 should be considered as digit numbers + float(value) + except ValueError: + # otherwise, they will be considered as formular injections + return not bool(re.compile(FileCheckConst.CSV_BLACK_LIST).search(value)) + return True + + if malicious_check: + for row in data: + for cell in row: + if not csv_value_is_valid(cell): + raise RuntimeError(f"Malicious value [{cell}] is not allowed " + f"to be written into the csv: {filepath}.") + + check_path_before_create(filepath) + file_path = os.path.realpath(filepath) + try: + with FileOpen(filepath, mode, encoding='utf-8-sig') as f: + writer = csv.writer(f) + writer.writerows(data) + except Exception as e: + logger.error(f'Save csv file "{os.path.basename(file_path)}" failed') + raise RuntimeError(f"Save csv file {file_path} failed.") from e + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + print(f"file_path:{file_path}") + + + +def write_csv_header(csv_path, header_func): + """如果是第一次写入，则写入 CSV 表头""" + header = header_func() # 获取表头 + logger.debug(f"Writing CSV header: {header}") + write_csv([header], csv_path, mode="a+") + + +def get_result_csv_header(): + """获取结果 CSV 文件的表头""" + return [ + MsCompareConst.DETAIL_CSV_API_NAME, + MsCompareConst.RESULT_CSV_FORWARD_TEST_SUCCESS, + MsCompareConst.RESULT_CSV_BACKWARD_TEST_SUCCESS, + MsCompareConst.DETAIL_CSV_MESSAGE, + ] + + +def get_detail_csv_header(): + """获取详细 CSV 文件的表头""" + detail_csv_header_basic_info = [ + MsCompareConst.DETAIL_CSV_API_NAME, + MsCompareConst.DETAIL_CSV_BENCH_DTYPE, + MsCompareConst.DETAIL_CSV_TESTED_DTYPE, + MsCompareConst.DETAIL_CSV_SHAPE, + ] + detail_csv_header_compare_result = list(compare_algorithms.keys()) + detail_csv_header_status = [ + MsCompareConst.DETAIL_CSV_PASS_STATUS, + MsCompareConst.DETAIL_CSV_MESSAGE, + ] + return detail_csv_header_basic_info + detail_csv_header_compare_result + detail_csv_header_status + + +def check_csv_header(headers, required_constants, csv_path): + """校验 CSV 文件表头是否包含所有必需的常量""" + missing_constants = [const for const in required_constants if not any(const in header for header in headers)] + + if missing_constants: + raise MsprobeBaseException( + MsprobeBaseException.MISSING_HEADER_ERROR, + f"{csv_path} 缺少以下必需的表头字段: {missing_constants}" + ) +def add_time_as_suffix(name): + return '{}_{}.csv'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) + + +# ======= 结果落盘管理类 ======== + +class DataManager: + def __init__(self, csv_dir, result_csv_path): + self.results = {} + self.results_exception_skip = {} + self.is_first_write = True # 标记用于添加表头 + self.csv_dir = csv_dir + self.api_names_set = set() # 存储已经出现的 API 名称的集合 + # 如果传入了 result_csv_path，则启用断点续检 + if result_csv_path: + self.resume_from_last_csv(result_csv_path) + self.initialize_api_names_set(result_csv_path) + else: + # 默认情况下，设置输出路径为空，等待首次写入时初始化 + self.result_out_path = os.path.join(self.csv_dir, add_time_as_suffix(MsCompareConst.RESULT_CSV_FILE_NAME)) + self.detail_out_path = os.path.join( + self.csv_dir, + os.path.basename(self.result_out_path).replace("result", "details") + ) + + if self.detail_out_path and os.path.exists(self.detail_out_path): + check_file_or_directory_path(self.detail_out_path) + + if self.result_out_path and os.path.exists(self.result_out_path): + check_file_or_directory_path(self.result_out_path) + + def initialize_api_names_set(self, result_csv_path): + """读取现有的 CSV 文件并存储已经出现的 API 名称到集合中""" + # 使用新的 read_csv 函数读取数据 + csv_data = read_csv(result_csv_path, as_pd=False) + + # 读取标题行 + headers = csv_data[0] if csv_data else [] # 如果文件为空，则 headers 会为空 + + # 使用提取的表头校验函数 + if check_csv_header(headers, get_result_csv_header(), result_csv_path): + + # 获取 "API Name" 列的索引 + api_name_index = None + for i, header in enumerate(headers): + if MsCompareConst.DETAIL_CSV_API_NAME in header: # CSV 文件的标题行包含了字节顺序标记,所以使用通过包含方式来查找 + api_name_index = i + break + + if api_name_index is None: + logger.warning(f"{result_csv_path} No column contains 'API Name'.") + return + + # 读取每一行的 API 名称 + for row in csv_data[1:]: # 跳过标题行，从第二行开始 + if row and len(row) > api_name_index: + api_name = row[api_name_index] + if api_name: + self.api_names_set.add(api_name) + + logger.debug(f"Initialized API names set from existing CSV: {self.api_names_set}") + + def is_unique_api(self, api_name): + """检查 API 名称是否唯一，如果已经存在则返回 False，否则加入集合并返回 True""" + if api_name in self.api_names_set: + return False + self.api_names_set.add(api_name) + return True + + def resume_from_last_csv(self, result_csv_path): + """从上次运行的 result_csv_path 恢复断点""" + # 获取上次的目录路径 + last_dir = os.path.dirname(result_csv_path) + + # 设置当前目录和输出路径，确保在首次写入时使用 + self.csv_dir = last_dir + self.detail_out_path = os.path.join(last_dir, os.path.basename(result_csv_path).replace("result", "details")) + if self.detail_out_path and os.path.exists(self.detail_out_path): + check_file_or_directory_path(self.detail_out_path) + self.result_out_path = result_csv_path + self.is_first_write = False + + def save_results(self, api_name_str): + if self.is_first_write: + # 直接写入表头 + logger.info("Writing CSV headers for the first time.") + write_csv_header(self.detail_out_path, get_detail_csv_header) + write_csv_header(self.result_out_path, get_result_csv_header) + self.is_first_write = False # 写入后标记为 False，避免重复写入表头 + + """写入详细输出和结果摘要并清理结果""" + logger.debug("Starting to write detailed output to CSV.") + self.to_detail_csv(self.detail_out_path) + logger.debug(f"Detailed output for {api_name_str} written to {self.detail_out_path}.") + + logger.debug("Starting to write result summary to CSV.") + self.to_result_csv(self.result_out_path) + logger.debug(f"Result summary for {api_name_str} written to {self.result_out_path}.") + + # 清理记录，准备下一次调用 + self.clear_results() + + def record(self, output_list): + if output_list is None: + return + for output in output_list: + api_real_name, forward_or_backward, basic_info, compare_result_dict = output + key = (api_real_name, forward_or_backward) + if key not in self.results: + self.results[key] = [] + self.results[key].append((basic_info, compare_result_dict)) + logger.debug(f"Complete self.results after recording: {self.results}") + + def record_exception_skip(self, api_name, forward_or_backward, err_msg): + ''' + record exception_skip information into self.record_exception_skip. + self.record_exception_skip: dict{str: dict{"forward": str/None, "backward": str/None}} + string in key is api_name, string in value is err_msg + ''' + if api_name not in self.results_exception_skip: + self.results_exception_skip[api_name] = {Const.FORWARD: None, Const.BACKWARD: None} + self.results_exception_skip[api_name][forward_or_backward] = err_msg + + def clear_results(self): + """清空 self.results 数据""" + logger.debug("Clearing self.results data.") + self.results.clear() + self.results_exception_skip.clear() + + def to_detail_csv(self, csv_path): + logger.debug("Preparing detail CSV headers and rows.") + detail_csv = [] + + detail_csv_header_compare_result = list(compare_algorithms.keys()) + + for _, results in self.results.items(): + for res in results: + basic_info, compare_result_dict = res + csv_row_basic_info = [ + basic_info.api_name, + basic_info.bench_dtype, + basic_info.tested_dtype, + basic_info.shape + ] + csv_row_compare_result = [ + compare_result_dict.get(algorithm_name).compare_value + for algorithm_name in detail_csv_header_compare_result + ] + csv_row_status = [basic_info.status, basic_info.err_msg] + csv_row = csv_row_basic_info + csv_row_compare_result + csv_row_status + detail_csv.append(csv_row) + logger.debug(f"Detail CSV row added: {csv_row}") + + logger.debug(f"Writing detail CSV to {csv_path}.") + write_csv(detail_csv, csv_path, mode="a+") + logger.debug(f"Detail CSV written successfully to {csv_path}.") + + def to_result_csv(self, csv_path): + ''' + depend on both self.results and self.results_exception_skip + ''' + logger.debug("Preparing result CSV data.") + result_csv = [] + + result_csv_dict = {} + for key, results in self.results.items(): + api_real_name, forward_or_backward = key + pass_status = CompareConst.PASS + overall_err_msg = "" + + for res in results: + basic_info, _ = res + if basic_info.status != CompareConst.PASS: + pass_status = CompareConst.ERROR + overall_err_msg += basic_info.err_msg + + overall_err_msg = "" if pass_status == CompareConst.PASS else overall_err_msg + + if api_real_name not in result_csv_dict: + result_csv_dict[api_real_name] = ResultCsvEntry() + if forward_or_backward == Const.FORWARD: + result_csv_dict[api_real_name].forward_pass_status = pass_status + result_csv_dict[api_real_name].forward_err_msg = overall_err_msg + else: + result_csv_dict[api_real_name].backward_pass_status = pass_status + result_csv_dict[api_real_name].backward_err_msg = overall_err_msg + + for api_name, entry in result_csv_dict.items(): + overall_err_msg = "" if (entry.forward_pass_status == CompareConst.PASS and + entry.backward_pass_status == CompareConst.PASS) else \ + entry.forward_err_msg + entry.backward_err_msg + row = [ + api_name, + entry.forward_pass_status, + entry.backward_pass_status, + overall_err_msg + ] + # change row if this api has exception_skip information + if api_name in self.results_exception_skip: + if self.results_exception_skip[api_name][Const.FORWARD] is not None: + row[1] = CompareConst.SKIP + row[-1] += self.results_exception_skip[api_name][Const.FORWARD] + if self.results_exception_skip[api_name][Const.BACKWARD] is not None: + row[2] = CompareConst.SKIP + row[-1] += self.results_exception_skip[api_name][Const.BACKWARD] + del self.results_exception_skip[api_name] + result_csv.append(row) + logger.debug(f"Result CSV row added: {row}") + for api_name in self.results_exception_skip: + current_exception_skip = self.results_exception_skip[api_name] + forward_status = None + backward_status = None + err_msg = "" + if current_exception_skip[Const.FORWARD] is not None: + forward_status = CompareConst.SKIP + err_msg += current_exception_skip[Const.FORWARD] + if current_exception_skip[Const.BACKWARD] is not None: + backward_status = CompareConst.SKIP + err_msg += current_exception_skip[Const.BACKWARD] + row = [api_name, forward_status, backward_status, err_msg] + result_csv.append(row) + + write_csv(result_csv, csv_path, mode="a+") + logger.debug(f"Result CSV written successfully to {csv_path}.") + + # 设置标记为 False，防止后续重复添加表头 + self.is_first_write = False + +# ======== 全局变量类 ======= + +class GlobalContext: + def __init__(self): + self.is_constructed = True + self.dump_data_dir = "" + self.framework = Const.MS_FRAMEWORK + + def init(self, is_constructed, dump_data_dir, framework): + self.is_constructed = is_constructed + self.dump_data_dir = dump_data_dir + self.framework = framework + + def get_dump_data_dir(self): + return self.dump_data_dir + + def get_is_constructed(self): + return self.is_constructed + + def get_framework(self): + return self.framework + + +global_context = GlobalContext() + +# ======== 输入类型类 ======= + +class ApiInputAggregation: + def __init__(self, inputs, kwargs, gradient_inputs) -> None: + """ + Args: + inputs: List[ComputeElement] + kwargs: dict{str: ComputeElement} + gradient_inputs: Union[List[ComputeElement], None] + """ + self.inputs = inputs + self.kwargs = kwargs + self.gradient_inputs = gradient_inputs + + +api_parent_module_mapping = { + (MsCompareConst.MINT, Const.MS_FRAMEWORK): mindspore.mint, + (MsCompareConst.MINT, Const.PT_FRAMEWORK): torch, + (MsCompareConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional, + (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional, + (MsCompareConst.TENSOR_API, Const.MS_FRAMEWORK): mindspore.Tensor, + (MsCompareConst.TENSOR_API, Const.PT_FRAMEWORK): torch.Tensor, + (MsCompareConst.MINDTORCH_TENSOR, Const.MT_FRAMEWORK): mindtorch_tensor, + (MsCompareConst.MINDTORCH_TENSOR, Const.PT_FRAMEWORK): torch.Tensor, + (MsCompareConst.MINDTORCH, Const.MT_FRAMEWORK): mindtorch, + (MsCompareConst.MINDTORCH, Const.PT_FRAMEWORK): torch, + (MsCompareConst.MINDTORCH_FUNC, Const.MT_FRAMEWORK): mindtorch_func, + (MsCompareConst.MINDTORCH_FUNC, Const.PT_FRAMEWORK): torch.nn.functional, + (MsCompareConst.MINDTORCH_DIST, Const.MT_FRAMEWORK): mindtorch_dist, + (MsCompareConst.MINDTORCH_DIST, Const.PT_FRAMEWORK): torch.distributed, + (MsCompareConst.FUNCTIONAL_API, Const.MS_FRAMEWORK): mindspore.ops + +} + + +api_parent_module_str_mapping = { + (MsCompareConst.MINT, Const.MS_FRAMEWORK): "mindspore.mint", + (MsCompareConst.MINT, Const.PT_FRAMEWORK): "torch", + (MsCompareConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): "mindspore.mint.nn.functional", + (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): "torch.nn.functional", + (MsCompareConst.TENSOR_API, Const.MS_FRAMEWORK): "mindspore.Tensor", + (MsCompareConst.TENSOR_API, Const.PT_FRAMEWORK): "torch.Tensor", + (MsCompareConst.MINDTORCH_TENSOR, Const.MT_FRAMEWORK): "mindtorch_tensor", + (MsCompareConst.MINDTORCH_TENSOR, Const.PT_FRAMEWORK): "torch.Tensor", + (MsCompareConst.MINDTORCH, Const.MT_FRAMEWORK): "mindtorch", + (MsCompareConst.MINDTORCH, Const.PT_FRAMEWORK): "torch", + (MsCompareConst.MINDTORCH_FUNC, Const.MT_FRAMEWORK): "mindtorch_func", + (MsCompareConst.MINDTORCH_FUNC, Const.PT_FRAMEWORK): "torch.nn.functional", + (MsCompareConst.MINDTORCH_DIST, Const.MT_FRAMEWORK): "mindtorch_dist", + (MsCompareConst.MINDTORCH_DIST, Const.PT_FRAMEWORK): "torch.distributed", + (MsCompareConst.FUNCTIONAL_API, Const.MS_FRAMEWORK): "mindspore.ops" +} + + +class ApiRunner: + def __call__(self, api_input_aggregation, api_name_str, forward_or_backward=Const.FORWARD, + api_platform=Const.MS_FRAMEWORK): + ''' + Args: + api_input_aggregation: ApiInputAggregation + api_name_str: str, e.g. "MintFunctional.relu.0" + forward_or_backward: str, Union["forward", "backward"] + api_platform: str, Union["mindspore", "torch", "mindtorch"] + + Return: + outputs: list[ComputeElement] + + Description: + run mindspore.mint/torch api + ''' + + api_type_str, api_sub_name = self.get_info_from_name(api_name_str, api_platform) + api_instance = self.get_api_instance(api_type_str, api_sub_name, api_platform) + + return self.run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform) + + @staticmethod + def get_info_from_name(api_name_str, api_platform=Const.MS_FRAMEWORK): + """ + Args: + api_name_str: str, the trimmed key of data dict in api_info.json. e.g. "MintFunctional.relu.0" + api_platform: str, the platform for the API, which can be either "mindspore" or "mindtorch". + It specifies which framework is being used. Default is "mindspore". + Return: + api_type_str: str, Union["MintFunctional", "Mint", "Tensor", "Torch", "Functional"] + api_sub_name: str, e.g. "relu" + """ + api_name_list = api_name_str.split(Const.SEP) + if len(api_name_list) != 3: + err_msg = f"ApiRunner.get_info_from_name failed: api_name_str: {api_name_str} is not in defined format" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) + api_type_str, api_sub_name = api_name_list[0], api_name_list[1] + if api_type_str not in [MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL, MsCompareConst.TENSOR_API, + MsCompareConst.FUNCTIONAL_API] \ + and api_platform == Const.MS_FRAMEWORK: + err_msg = f"ApiRunner.get_info_from_name failed: not mint, mint.nn.functional or Tensor api" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) + + if api_type_str not in MsCompareConst.MT_VALID_API_TYPES and api_platform == Const.MT_FRAMEWORK: + err_msg = f"ApiRunner.get_info_from_name failed: not torch, functional or Tensor api" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) + return api_type_str, api_sub_name + + @staticmethod + def get_api_instance(api_type_str, api_sub_name, api_platform): + """ + Args: + api_type_str: str, Union["MintFunctional", "Mint", "Tensor", "Functional"] + api_sub_name: str, e.g. "relu" + api_platform: str: Union["mindspore", "pytorch"] + + Return: + api_instance: function object + + Description: + get mindspore.mint/torch api function + mindspore.mint.{api_sub_name} <--> torch.{api_sub_name} + mindspore.mint.nn.functional.{api_sub_name} <--> torch.nn.functional.{api_sub_name} + """ + + api_parent_module = api_parent_module_mapping.get((api_type_str, api_platform)) + api_parent_module_str = api_parent_module_str_mapping.get((api_type_str, api_platform)) + full_api_name = api_parent_module_str + Const.SEP + api_sub_name + + if not hasattr(api_parent_module, api_sub_name): + err_msg = f"ApiRunner.get_api_instance failed: {full_api_name} is not found" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ApiWrong)) + + api_instance = getattr(api_parent_module, api_sub_name) + if not callable(api_instance): + err_msg = f"ApiRunner.get_api_instance failed: {full_api_name} is not callable" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ApiWrong)) + + return api_instance + + @staticmethod + def run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform): + inputs = tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) + for compute_element in api_input_aggregation.inputs) + kwargs = {key: value.get_parameter(get_origin=False, tensor_platform=api_platform) + for key, value in api_input_aggregation.kwargs.items()} + gradient_inputs = api_input_aggregation.gradient_inputs + + if forward_or_backward == Const.FORWARD: + forward_result = api_instance(*inputs, **kwargs) # can be single tensor or tuple + forward_result_tuple = convert_to_tuple(forward_result) + res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in forward_result_tuple] + if api_platform == Const.MS_FRAMEWORK or api_platform == Const.MT_FRAMEWORK: + return res_compute_element_list, inputs, kwargs, forward_result_tuple + else: + if gradient_inputs is None: + err_msg = f"ApiRunner.run_api failed: run backward api but gradient_inputs is missing" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) + gradient_inputs = tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) + for compute_element in gradient_inputs) + if api_platform == Const.MS_FRAMEWORK or api_platform == Const.MT_FRAMEWORK: + if len(gradient_inputs) == 1: + gradient_inputs = gradient_inputs[0] + + def api_with_kwargs(*forward_inputs): + return api_instance(*forward_inputs, **kwargs) + + grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_with_kwargs) + backward_result = grad_func(*inputs, gradient_inputs) # can be single tensor or tuple + backward_result_tuple = convert_to_tuple(backward_result) + res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_tuple] + return res_compute_element_list, gradient_inputs, backward_result_tuple + else: + # set requires_grad + requires_grad_index = [] + for index, tensor in enumerate(inputs): + if isinstance(tensor, torch.Tensor) and \ + torch_dtype_to_dtype_str.get(tensor.dtype) in float_dtype_str_list: + setattr(tensor, "requires_grad", True) + requires_grad_index.append(index) + forward_results = api_instance(*inputs, **kwargs) + forward_results = convert_to_tuple(forward_results) + for forward_res, gradient_in in zip(forward_results, gradient_inputs): + forward_res.backward(gradient_in) + backward_result_list = [] + for index in requires_grad_index: + backward_result_list.append(getattr(inputs[index], "grad")) + res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_list] + + return res_compute_element_list + + +api_runner = ApiRunner() + +# ======== 数据结构类 ======== + +class ResultCsvEntry: + def __init__(self) -> None: + self.forward_pass_status = None + self.backward_pass_status = None + self.forward_err_msg = "" + self.backward_err_msg = "" + self.overall_err_msg = None + +class ProcessResultPacket: + def __init__(self, process_status, result, err_msg) -> None: + self.process_status = process_status + self.result = result + self.err_msg = err_msg + +class MstensorMetaData: + def __init__(self, dtype_str, npy_path, maximum, minimum, shape) -> None: + self.dtype_str = dtype_str + self.npy_path = npy_path + self.maximum = maximum + self.minimum = minimum + self.shape = shape + + +class DtypeMetaData: + def __init__(self, dtype_str) -> None: + self.dtype_str = dtype_str + + +class ComputeElement: + def __init__(self, compute_element_info=None, parameter=None): + self.supported_parameter_type = tuple(type_to_api_info_type_str.keys()) + tuple([torch.Tensor, tuple]) + if parameter is not None: + self._init_with_parameter(parameter) + elif isinstance(compute_element_info, (list, dict)): + self._init_from_compute_element_info(compute_element_info) + elif compute_element_info is None: + self._init_from_null_compute_element_info() + else: + pass + logger.error_log_with_exp( + "ComputeElement.__init__ failed: not init with parameter or compute_element info is not (list, dict)", + ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + + @staticmethod + def transfer_to_torch_tensor(ms_tensor): + ''' + Args: + ms_tensor: mindspore.Tensor + Return: + torch_tensor: torch.Tensor + ''' + ms_dtype = ms_tensor.dtype + dtype_str = ms_dtype_to_dtype_str.get(ms_dtype) + if dtype_str not in dtype_str_to_torch_dtype: + err_msg = f"ComputeElement.transfer_to_torch_tensor failed: no matching torch dtype for {dtype_str}" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + else: + torch_dtype = dtype_str_to_torch_dtype.get(dtype_str) + + if dtype_str in int_dtype_str_list: + middle_dtype = mindspore.int64 + else: + middle_dtype = mindspore.float64 + np_ndarray = ms_tensor.astype(middle_dtype).numpy() + torch_tensor = torch.from_numpy(np_ndarray).to(torch_dtype) + return torch_tensor + + @staticmethod + def transfer_to_mindtorch_tensor(ms_tensor): + """ + Args: + ms_tensor: mindspore.Tensor + Return: + mindtorch_tensor: mindtorch.Tensor + """ + + ms_dtype = ms_tensor.dtype + + dtype_str = ms_dtype_to_dtype_str.get(ms_dtype) + + if dtype_str not in dtype_str_to_mindtorch_dtype: + err_msg = f"ComputeElement.transfer_to_mindtorch_tensor failed: no matching mindtorch dtype for {dtype_str}" + logger.error_log_with_exp(err_msg, + ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + else: + mindtorch_dtype = dtype_str_to_mindtorch_dtype.get(dtype_str) + + if dtype_str in int_dtype_str_list: + middle_dtype = mindspore.int64 + else: + middle_dtype = mindspore.float64 + + np_ndarray = ms_tensor.astype(middle_dtype).numpy() + + mindtorch_tensor = mindtorch.from_numpy(np_ndarray).to(ms_dtype) + + return mindtorch_tensor + + @staticmethod + def transfer_to_mindspore_tensor(torch_tensor): + ''' + Args: + torch_tensor: torch.Tensor + + Return: + ms_tensor: mindspore.Tensor + ''' + torch_dtype = torch_tensor.dtype + dtype_str = torch_dtype_to_dtype_str.get(torch_dtype) + if dtype_str not in dtype_str_to_ms_dtype: + err_msg = \ + f"ComputeElement._transfer_to_mindspore_tensor failed: no matching mindspore dtype for {dtype_str}" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + else: + ms_dtype = dtype_str_to_ms_dtype.get(dtype_str) + + if dtype_str in int_dtype_str_list: + middle_dtype = torch.int64 + else: + middle_dtype = torch.float64 + np_ndarray = torch_tensor.to(middle_dtype, copy=True).numpy() + ms_tensor = mindspore.Tensor.from_numpy(np_ndarray).astype(ms_dtype) + return ms_tensor + + @staticmethod + def convert_inf_to_real_num(value, dtype_str): + if value == float("inf"): + np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE) + value = np.finfo(np_dtype).max + elif value == float("-inf"): + np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE) + value = np.finfo(np_dtype).min + return value + + def get_parameter(self, get_origin=True, tensor_platform=Const.MS_FRAMEWORK): + ''' + Args: + get_origin: boolean + tensor_platform: str, Union["mindspore", "pytorch"] + + Return: + parameter: Union[int, float, str, slice, tuple, torch.Tensor, mindspore.Tensor] + ''' + if self.parameter is None: + return self.parameter + if isinstance(self.parameter, tuple): + return tuple([compute_element.get_parameter(get_origin=get_origin, tensor_platform=tensor_platform) + for compute_element in self.parameter]) + elif isinstance(self.parameter, self.supported_parameter_type): + parameter_tmp = self.parameter + elif isinstance(self.parameter, DtypeMetaData): + if tensor_platform == Const.MS_FRAMEWORK: + parameter_tmp = dtype_str_to_ms_dtype.get(self.parameter.dtype_str) + elif tensor_platform == Const.PT_FRAMEWORK: + parameter_tmp = dtype_str_to_torch_dtype.get(self.parameter.dtype_str) + elif tensor_platform == Const.MT_FRAMEWORK: + parameter_tmp = dtype_str_to_mindtorch_dtype.get(self.parameter.dtype_str) + + elif isinstance(self.parameter, MstensorMetaData): + mstensor_meta_data = self.parameter + ms_dtype = dtype_str_to_ms_dtype.get(mstensor_meta_data.dtype_str) + if global_context.get_is_constructed(): + np_dtype = dtype_str_to_np_dtype.get(mstensor_meta_data.dtype_str, DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE) + ndarray = self._construct_ndarray(mstensor_meta_data.shape, mstensor_meta_data.maximum, + mstensor_meta_data.minimum, np_dtype) + else: + ndarray = load_npy(mstensor_meta_data.npy_path) + parameter_tmp = mindspore.Tensor(ndarray, dtype=ms_dtype) + else: + err_msg = "ComputeElement.get_parameter failed: self.parameter type is not in " \ + "(int, float, str, slice, bool, torch.Tensor, mindspore.Tensor, MstensorMetaData)" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + + # if necessary, do transfer + if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and tensor_platform == Const.PT_FRAMEWORK: + parameter = self.transfer_to_torch_tensor(parameter_tmp) + elif not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and tensor_platform == Const.MT_FRAMEWORK: + parameter = self.transfer_to_mindtorch_tensor(parameter_tmp) + elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and tensor_platform == Const.MS_FRAMEWORK: + parameter = self.transfer_to_mindspore_tensor(parameter_tmp) + else: + parameter = parameter_tmp + + return parameter + + def get_shape(self): + return self.shape + + def get_dtype(self): + return self.dtype_str + + def _construct_ndarray(self, shape, maximum, minimum, np_dtype): + shape = tuple(shape) + np.random.seed({random_seed}) + if np_dtype == np.bool_: + ndarray = np.random.rand(*shape) > 0.5 + else: + maximum = self.convert_inf_to_real_num(maximum, np_dtype) + minimum = self.convert_inf_to_real_num(minimum, np_dtype) + ndarray = np.random.uniform(minimum, maximum, shape).astype(np_dtype) + return ndarray + + def _init_from_null_compute_element_info(self): + self.parameter = None + self.shape = tuple() + self.dtype = "None" + + def _init_from_compute_element_info(self, compute_element_info): + ''' + Args: + compute_element_info: Union[list, dict] + + Return: + void + + init member attributes: self.shape, self.dtype_str, self.parameter + ''' + if isinstance(compute_element_info, list): + self.shape = tuple() + self.dtype_str = TUPLE_TYPE_STR + self.parameter = tuple([ComputeElement(compute_element_info=sub_info) + for sub_info in compute_element_info]) + else: + type_str = check_and_get_from_json_dict(compute_element_info, "type", "type field in api_info.json", + accepted_type=str, accepted_value=api_info_type_str_to_type.keys()) + self.shape = tuple() + self.dtype_str = type_str + if type_str == MINDSPORE_TENSOR_TYPE_STR: + self._init_from_mstensor_compute_element_info(compute_element_info) + else: + value = check_and_get_from_json_dict(compute_element_info, "value", "value field in api_info.json") + if type_str == MINDSPORE_DTYPE_TYPE_STR: + self.parameter = DtypeMetaData(value) + elif type_str == SLICE_TYPE_STR: + self.parameter = slice(*tuple(value)) + else: # type_str in ("str", "int", "float", "bool") + self.parameter = value + + def _init_from_mstensor_compute_element_info(self, compute_element_info): + ''' + do not load real tensor, only record meta data + ''' + dtype_str = check_and_get_from_json_dict(compute_element_info, "dtype", "dtype field in api_info.json", + accepted_type=str, accepted_value=dtype_str_to_ms_dtype.keys()) + shape = check_and_get_from_json_dict(compute_element_info, "shape", "shape field in api_info.json", + accepted_type=(list,)) + if global_context.get_is_constructed(): + maximum = check_and_get_from_json_dict(compute_element_info, "Max", "Max field in api_info.json", + accepted_type=(int, float)) + minimum = check_and_get_from_json_dict(compute_element_info, "Min", "Min field in api_info.json", + accepted_type=(int, float)) + + npy_path = None + else: + maximum, minimum = None, None + data_name = check_and_get_from_json_dict(compute_element_info, "data_name", + "data_name field in api_info.json", accepted_type=(str,)) + npy_path = os.path.join(global_context.get_dump_data_dir(), data_name) + mstensor_meta_data = MstensorMetaData(dtype_str, npy_path, maximum, minimum, shape) + self.parameter = mstensor_meta_data + self.dtype_str = dtype_str + self.shape = tuple(shape) + + def _init_with_parameter(self, parameter): + self.parameter = parameter + print(f"parameter:{parameter}") + print(f"self.supported_parameter_type:{self.supported_parameter_type}") + if isinstance(parameter, dict): + # 这里假设 dict 中有 'type'、'shape'、'dtype' 等字段 + return self._init_from_compute_element_info(parameter) + self.shape = tuple() + if not isinstance(parameter, self.supported_parameter_type): + err_msg = "ComputeElement._init_with_parameter failed: " \ + "parameter type is not in (int, float, str, slice, bool, torch.Tensor, mindspore.Tensor)" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + if isinstance(parameter, mindspore.Tensor): + self.shape = tuple(parameter.shape) + self.dtype_str = ms_dtype_to_dtype_str.get(parameter.dtype) + elif isinstance(parameter, torch.Tensor): + self.shape = tuple(parameter.shape) + self.dtype_str = torch_dtype_to_dtype_str.get(parameter.dtype) + elif isinstance(parameter, typing.Type): + self.dtype_str = MINDSPORE_DTYPE_TYPE_STR + self.parameter = DtypeMetaData(ms_dtype_to_dtype_str.get(parameter)) + elif isinstance(parameter, torch.dtype): + self.dtype_str = TORCH_DTYPE_TYPE_STR + self.parameter = DtypeMetaData(torch_dtype_to_dtype_str.get(parameter)) + elif isinstance(parameter, tuple): + self.dtype_str = TUPLE_TYPE_STR + self.parameter = tuple([ComputeElement(parameter=param) for param in parameter]) + else: + self.dtype_str = type_to_api_info_type_str.get(type(parameter)) + print(f"self.dtype_str{self.dtype_str}") + +class BasicInfoAndStatus: + def __init__(self, api_name, bench_dtype, tested_dtype, shape, status, err_msg) -> None: + self.api_name = api_name + self.bench_dtype = bench_dtype + self.tested_dtype = tested_dtype + self.shape = shape + self.status = status + self.err_msg = err_msg + +# ======== api执行类 ======= + +def get_input(propagation): + args_info_forward = {args_info_forward} + kwargs_info_forward = {kwargs_info_forward} + args_info_backward = {args_info_backward} + forward_inputs = [ComputeElement(compute_element_info=compute_element_info) + for compute_element_info in args_info_forward] + kwargs_compute_element_dict = { + key_str: ComputeElement(compute_element_info=compute_element_info) + for key_str, compute_element_info in kwargs_info_forward.items() + } + if args_info_backward: + gradient_inputs = [ComputeElement(compute_element_info=compute_element_info) + for compute_element_info in args_info_backward] + else: + gradient_inputs = None + return ApiInputAggregation( + forward_inputs, + kwargs_compute_element_dict, + gradient_inputs + ) + +# 运行和比对函数 +def run_and_compare_helper(api_name_str, api_input_aggregation, forward_or_backward): + """ + Args: + api_info: ApiInfo + api_name_str: str + api_input_aggregation: ApiInputAggregation + forward_or_backward: str: Union["forward", "backward"] + + Return: + output_list: List[tuple(str, str, BasicInfoAndStatus, dict{str: CompareResult})] + + Description: + get mindspore api output, run torch api and get output. + compare output. + record compare result. + """ + # get output + if forward_or_backward == Const.FORWARD: + tested_outputs, inputs, kwargs, forward_result_tuple = api_runner(api_input_aggregation, api_name_str, + forward_or_backward, + global_context.get_framework()) + print(f"inputs:{inputs}") + print(f"kwargs:{kwargs}") + print(f"forward_result_tuple:{forward_result_tuple}") + elif forward_or_backward == Const.BACKWARD: + tested_outputs, gradient_inputs, backward_result_tuple = api_runner(api_input_aggregation, api_name_str, + forward_or_backward, + global_context.get_framework()) + print(f"gradient_inputs:{gradient_inputs}") + print(f"backward_result_tuple:{backward_result_tuple}") + else: + tested_outputs = api_runner(api_input_aggregation, api_name_str, + forward_or_backward, global_context.get_framework()) + + bench_outputs = api_runner(api_input_aggregation, api_name_str, forward_or_backward, Const.PT_FRAMEWORK) + + tested_outputs = trim_output_compute_element_list(tested_outputs, forward_or_backward) + bench_outputs = trim_output_compute_element_list(bench_outputs, forward_or_backward) + + # compare output + output_list = [] + for i, (bench_out, tested_out) in enumerate(zip(bench_outputs, tested_outputs)): + api_name_with_slot = Const.SEP.join([api_name_str, forward_or_backward, Const.OUTPUT, str(i)]) + bench_dtype = bench_out.get_dtype() + tested_dtype = tested_out.get_dtype() + shape = bench_out.get_shape() + + compare_result_dict = dict() + for compare_algorithm_name, compare_algorithm in compare_algorithms.items(): + compare_result = compare_algorithm(bench_out, tested_out) + compare_result_dict[compare_algorithm_name] = compare_result + + if compare_result_dict.get(CompareConst.COSINE).pass_status == CompareConst.PASS and \ + compare_result_dict.get(CompareConst.MAX_ABS_ERR).pass_status == CompareConst.PASS: + status = CompareConst.PASS + err_msg = "" + else: + status = CompareConst.ERROR + err_msg = (compare_result_dict.get(CompareConst.COSINE).err_msg + + compare_result_dict.get(CompareConst.MAX_ABS_ERR).err_msg) + + # self.pre_forward_hook(api_name_str, None, inputs, kwargs) + basic_info_status = \ + BasicInfoAndStatus(api_name_with_slot, bench_dtype, tested_dtype, shape, status, err_msg) + output_list.append(tuple([api_name_str, forward_or_backward, basic_info_status, compare_result_dict])) + return output_list + + +if __name__ == "__main__": + framework = "{framework}" + dump_data_dir = "{real_data_path}" + api_name = "{api_name}" + api_full_name = "{api_full_name}" + api_name_str = ".".join(api_full_name.split(".")[:3]) + propagation = "{propagation}" + data_mode = "{data_mode}" + torch.manual_seed({random_seed}) + + data_manager = DataManager("./op_result_output", None) + create_directory("./op_result_output") + + is_constructed = data_mode == "random_data" + global_context.init(is_constructed, dump_data_dir, framework) + + for i in range({iter_times}): + print(f"iter: {{i}}:") + if propagation == BACKWARD: + + + backward_inputs_aggregation = get_input(propagation) + + backward_output_list = run_and_compare_helper(api_name_str, backward_inputs_aggregation, + Const.BACKWARD) + process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.SUCCESS, + result=backward_output_list, err_msg="") + + + if process_result_packet.process_status is MsCompareConst.ProcessStatus.SUCCESS: + data_manager.record(process_result_packet.result) + elif process_result_packet.process_status == MsCompareConst.ProcessStatus.EXCEPTION_SKIP: + data_manager.record_exception_skip(api_name_str, Const.BACKWARD, process_result_packet.err_msg) + + data_manager.save_results(api_name_str) + else: + forward_inputs_aggregation = get_input(propagation) + + forward_output_list = run_and_compare_helper(api_name_str, forward_inputs_aggregation, + Const.FORWARD) + process_result_packet = ProcessResultPacket(process_status=MsCompareConst.ProcessStatus.SUCCESS, + result=forward_output_list, err_msg="") + + + if process_result_packet.process_status is MsCompareConst.ProcessStatus.SUCCESS: + data_manager.record(process_result_packet.result) + elif process_result_packet.process_status == MsCompareConst.ProcessStatus.EXCEPTION_SKIP: + data_manager.record_exception_skip(api_name_str, Const.FORWARD, process_result_packet.err_msg) + + data_manager.save_results(api_name_str) + + print("Compare finished.") \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py index 1913675ad162bf690fc0aed5fc84c245ae4f73ca..37f6faa514eaf4855211f9db8ff45982c3b8b976 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/multi_api_accuracy_checker.py @@ -33,6 +33,9 @@ from msprobe.mindspore.api_accuracy_checker.multi_data_manager import MultiDataM from msprobe.mindspore.common.log import logger from msprobe.mindspore.common.const import MsCompareConst +from msprobe.core.data_dump.data_collector import build_data_collector +from msprobe.core.common.utils import Const, print_tools_ends_info, DumpPathAggregation + class MultiApiAccuracyChecker(ApiAccuracyChecker): def __init__(self, args): @@ -51,6 +54,12 @@ class MultiApiAccuracyChecker(ApiAccuracyChecker): # 初始化一个属性来存储当前的设备ID（用于日志中显示） self.current_device_id = None + self.save_error_data = args.save_error_data + if self.save_error_data: + config, dump_path_aggregation = self.init_save_error_data(args) + self.data_collector = build_data_collector(config) + self.data_collector.update_dump_paths(dump_path_aggregation) + def process_on_device(self, device_id, api_infos, progress_queue): """ 在特定设备上处理一部分API。 diff --git a/debug/accuracy_tools/msprobe/mindspore/cell_processor.py b/debug/accuracy_tools/msprobe/mindspore/cell_processor.py index 6dc5d510ef51ab2a135a8bdf9f15ac670fba9e56..2c3426192c1d93a6b2276f6b3dc441273bac8473 100644 --- a/debug/accuracy_tools/msprobe/mindspore/cell_processor.py +++ b/debug/accuracy_tools/msprobe/mindspore/cell_processor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,21 +13,50 @@ # See the License for the specific language governing permissions and # limitations under the License. -from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope +from collections import OrderedDict + +from mindspore import Tensor +from mindspore.common.hook_handle import HookHandle +from mindspore.ops.operations import _inner_ops as inner + from msprobe.core.common.const import Const +from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.data_dump.scope import ModuleRangeScope, MixRangeScope, BaseScope +from msprobe.mindspore.common.const import Const as MsConst +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.common.utils import ( + is_mindtorch, + get_cells_and_names_with_index, + has_kwargs_in_forward_hook, + is_graph_mode_cell_dump_allowed +) +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.mindspore.dump.graph_mode_cell_dump import GraphModeCellDump +from msprobe.mindspore.runtime import Runtime + + +def get_cell_construct(construct): + def _construct(self, *args, **kwargs): + if hasattr(self, 'msprobe_hook'): + setattr(self, 'msprobe_input_kwargs', kwargs) + return construct(self, *args, **kwargs) + return _construct class CellProcessor: cell_count = {} cell_stack = [] - api_parent_node = "" + api_parent_node = None module_node = {} + cell_bw_hook_kernels = {} + cell_backward_pre_hook = [] + cell_backward_hook = [] def __init__(self, scope): self.scope = scope if isinstance(scope, (ModuleRangeScope, MixRangeScope)) else None @staticmethod - def set_cell_count(cell_name): + def set_and_get_calls_number(cell_name): if cell_name not in CellProcessor.cell_count: CellProcessor.cell_count[cell_name] = 0 else: @@ -38,42 +67,185 @@ class CellProcessor: def reset_cell_stats(cls): cls.cell_count = {} cls.cell_stack = [] - cls.api_parent_node = "" + cls.api_parent_node = None cls.module_node = {} + cls.cell_bw_hook_kernels = {} + cls.cell_backward_pre_hook = [] + cls.cell_backward_hook = [] - def node_hook(self, name_prefix, start_or_stop, **kwargs): - def begin_hook(cell, input_data): - full_name = self.set_and_get_reserved_name(cell, name_prefix, is_called_by_pre_hook=True) - if CellProcessor.cell_stack: - CellProcessor.module_node[full_name] = CellProcessor.cell_stack[-1] - else: - CellProcessor.module_node[full_name] = None + def register_cell_hook(self, models, build_hook, config: DebuggerConfig): + if not models: + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, + 'The model cannot be None, when level is "L0" or "mix"') + + is_registered = False + model_type = Const.MODULE if is_mindtorch() else Const.CELL + cells_with_index_in_pynative_mode, cells_with_index_in_graph_mode = get_cells_and_names_with_index(models) + construct_name = '_call_impl' if is_mindtorch() else '_run_construct' + + for index, cells_and_names in cells_with_index_in_pynative_mode.items(): + model = models if index == "-1" else models[int(index)] + for name, cell in cells_and_names: + if cell == model: + continue + + if not has_kwargs_in_forward_hook(): + if not hasattr(cell.__class__, 'msprobe_construct'): + setattr(cell.__class__, 'msprobe_construct', True) + if hasattr(cell.__class__, construct_name): + setattr(cell.__class__, construct_name, + get_cell_construct(getattr(cell.__class__, construct_name))) + setattr(cell, 'msprobe_hook', True) + + cell_index = (index + Const.SEP) if index != "-1" else "" + prefix = f'{model_type}{Const.SEP}{cell_index}{name}{Const.SEP}{cell.__class__.__name__}{Const.SEP}' + + forward_pre_hook = self.build_cell_hook(prefix, build_hook) + cell.register_forward_pre_hook(forward_pre_hook) + + if not is_registered: + logger.info("The cell hook function is successfully mounted to the model.") + is_registered = True + + if is_graph_mode_cell_dump_allowed(config): + cells_and_names_in_graph_mode = [] + for index, cells_and_names in cells_with_index_in_graph_mode.items(): + model = models if index == "-1" else models[int(index)] + for name, cell in cells_and_names: + if cell == model: + continue + cell_index = (index + Const.SEP) if index != "-1" else "" + cells_and_names_in_graph_mode.append((f'{cell_index}{name}', cell)) + + if cells_and_names_in_graph_mode: + Runtime.run_mode = MsConst.PYNATIVE_GRAPH_MODE + GraphModeCellDump(config, cells_and_names_in_graph_mode, strict=False).handle() - CellProcessor.cell_stack.append(full_name) - CellProcessor.api_parent_node = full_name + def build_cell_hook(self, cell_name, build_data_hook): + def forward_pre_hook(cell, args): + index = CellProcessor.set_and_get_calls_number(cell_name) + full_forward_name = f'{cell_name}{Const.FORWARD}{Const.SEP}{index}' + full_backward_name = f'{cell_name}{Const.BACKWARD}{Const.SEP}{index}' - if self.scope: - self.scope.begin_module(full_name) + self.set_construct_info_in_pre_hook(full_forward_name) - def end_hook(cell, input_data, output_data): - if CellProcessor.cell_stack: - CellProcessor.cell_stack.pop() - if CellProcessor.cell_stack: - CellProcessor.api_parent_node = CellProcessor.cell_stack[-1] + if not hasattr(cell, 'msprobe_forward_hook'): + if is_mindtorch(): + cell.register_forward_hook(forward_hook, prepend=True, with_kwargs=True) + else: + forward_hook_dict = getattr(cell, '_forward_hook', OrderedDict()) + if has_kwargs_in_forward_hook(): + forward_hook_with_kwargs_dict = getattr(cell, '_forward_hook_with_kwargs', OrderedDict()) + handle = HookHandle(forward_hook_dict, extra_dict=forward_hook_with_kwargs_dict) + forward_hook_with_kwargs_dict[handle.handle_id] = True + else: + handle = HookHandle(forward_hook_dict) + forward_hook_dict[handle.handle_id] = forward_hook + forward_hook_dict.move_to_end(handle.handle_id, last=False) + + setattr(cell, 'msprobe_forward_hook', True) + + def get_backward_hook(backward_data_hook, full_backward_name): + def backward_hook_fn(cell, grad_input, grad_output): + new_output = backward_data_hook(cell, grad_input, grad_output) + self.set_construct_info_in_hook(full_backward_name) + cell.has_pre_hook_called = False + return new_output + return backward_hook_fn + + enable_hooked = sum( + [isinstance(ele, Tensor) and ele.dtype not in MsConst.NonDifferentiableType for ele in args] + ) + if enable_hooked: + backward_hook = OrderedDict() + _, _, backward_data_hook, _ = build_data_hook(BaseScope.Module_Type_Module, full_forward_name) + backward_hook[full_backward_name] = get_backward_hook(backward_data_hook, full_backward_name) + CellProcessor.cell_backward_hook.append(backward_hook) + bw_hook = inner.CellBackwardHook(full_backward_name, cell, + self.cell_backward_hook[-1]) + bw_hook.register_backward_hook() + CellProcessor.cell_bw_hook_kernels[full_forward_name] = bw_hook + + args = bw_hook(*args) + + return args + + def forward_hook(cell, args, kwargs_or_output, output_or_kwargs=None): + index = CellProcessor.cell_count.get(cell_name, 0) + full_forward_name = f'{cell_name}{Const.FORWARD}{Const.SEP}{index}' + full_backward_name = f'{cell_name}{Const.BACKWARD}{Const.SEP}{index}' + + self.set_construct_info_in_hook(full_forward_name) + + _, forward_data_hook, backward_data_hook, _ = build_data_hook(BaseScope.Module_Type_Module, + full_forward_name) + hook_result = forward_data_hook(cell, args, kwargs_or_output, output_or_kwargs) + if hook_result is not None: + outputs = hook_result else: - CellProcessor.api_parent_node = None + outputs = output_or_kwargs if has_kwargs_in_forward_hook() else kwargs_or_output + + bw_hook = CellProcessor.cell_bw_hook_kernels.get(full_forward_name) + if bw_hook: + if not isinstance(outputs, (Tensor, tuple)): + logger.warning("For backward hooks to be called," + " cell output should be a Tensor or a tuple of Tensors" + f" but received {type(outputs)}") + if isinstance(outputs, tuple): + new_outputs = bw_hook(*outputs) + else: + new_outputs = bw_hook(outputs) + if isinstance(outputs, tuple) and len(outputs) == 1: + new_outputs = (new_outputs,) + outputs = new_outputs + + def get_backward_pre_hook(full_backward_name, backward_data_hook): + def backward_pre_hook_fn(cell, grad_output): + cell.has_pre_hook_called = True + self.set_construct_info_in_pre_hook(full_backward_name) + if backward_data_hook: + backward_data_hook(cell, (), grad_output) + self.set_construct_info_in_hook(full_backward_name) + cell.has_pre_hook_called = False + return backward_pre_hook_fn - if self.scope: - self.scope.end_module(cell.mindstudio_reserved_name) + backward_pre_hook = OrderedDict() + backward_data_hook = None if bw_hook else backward_data_hook + backward_pre_hook[full_backward_name] = get_backward_pre_hook(full_backward_name, backward_data_hook) + CellProcessor.cell_backward_pre_hook.append(backward_pre_hook) + bw_pre_hook = inner.CellBackwardHook(full_backward_name, cell, + self.cell_backward_pre_hook[-1]) + bw_pre_hook.register_backward_pre_hook() - return begin_hook if Const.START == start_or_stop else end_hook + if isinstance(outputs, tuple): + result = bw_pre_hook(*outputs) + else: + result = bw_pre_hook(outputs) + if isinstance(outputs, tuple): + if len(outputs) == 1: + result = (result,) + if len(result) != len(outputs): + raise TypeError( + f"The backward pre hook return value size is {len(result)} " + f"not equal to output size {len(outputs)}" + ) + return result + + return forward_pre_hook - def set_and_get_reserved_name(self, cell, cell_name, is_called_by_pre_hook=False): - if not is_called_by_pre_hook and hasattr(cell, 'has_pre_hook_called') and cell.has_pre_hook_called: - cell.has_pre_hook_called = False + def set_construct_info_in_pre_hook(self, full_name): + if self.cell_stack: + CellProcessor.module_node[full_name] = self.cell_stack[-1] else: - if is_called_by_pre_hook: - cell.has_pre_hook_called = True - index = self.set_cell_count(cell_name) - cell.mindstudio_reserved_name = cell_name + Const.SEP + str(index) - return cell.mindstudio_reserved_name + CellProcessor.module_node[full_name] = None + CellProcessor.cell_stack.append(full_name) + CellProcessor.api_parent_node = full_name + if self.scope: + self.scope.begin_module(full_name) + + def set_construct_info_in_hook(self, full_name): + if self.cell_stack: + CellProcessor.cell_stack.pop() + CellProcessor.api_parent_node = CellProcessor.cell_stack[-1] if self.cell_stack else None + if self.scope: + self.scope.end_module(full_name) diff --git a/debug/accuracy_tools/msprobe/mindspore/code_mapping/graph_parser.py b/debug/accuracy_tools/msprobe/mindspore/code_mapping/graph_parser.py index ee35750fb35c100e2025b0dcbdd9e20ef998b2ee..e09178d6dce5da7adc382f7ee62e8e32fca4aac4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/code_mapping/graph_parser.py +++ b/debug/accuracy_tools/msprobe/mindspore/code_mapping/graph_parser.py @@ -34,19 +34,6 @@ class Parser: if isinstance(subgraph_node.attrs, list): subgraph_node.attrs.extend(attrs) - @staticmethod - def parse_graph_attributes(text: str, graph_node: GraphNode) -> None: - attr_pattern = re.compile(r'# Attrs:\s*(.*)', re.DOTALL) - match = attr_pattern.search(text, graph_node.pos) - if match: - attrs = match.group(1).strip().split('\n') - for attr in attrs: - if not attr: - break - key, value = attr.split(':') - if isinstance(graph_node.attrs, dict): - graph_node.attrs[key.strip()] = value.strip() - @staticmethod def parse_code_info(text: str, start_pos: int, end_pos: int) -> List[str]: code_info = [] @@ -124,8 +111,9 @@ class Parser: scope_match = scope_pattern.search(text, end_pos) scope = scope_match.group(1) if scope_match else "" - id_pattern = re.compile(r'.*cnode_primal_attrs:' - r'\s*\{.*\b(?:forward_unique_id|unique_id):\s*\"(\d+)\".*', re.IGNORECASE) + id_pattern = re.compile( + r'cnode_primal_attrs:'r'\s*\{[\w+]{1, 10000}\b(?:forward_unique_id|unique_id):\s*\"(\d+)\"', + re.IGNORECASE) unique_id_match = id_pattern.search(text, end_pos, scope_match.start()) unique_id = unique_id_match.group(1) if unique_id_match else None @@ -186,7 +174,7 @@ class Parser: node_info.var_inputs.append(callee_name) def parse_subgraphs(self, text: str) -> None: - subgraph_pattern = re.compile(r'subgraph\s+@(\S+)(\([^\)]*\))?\s+.*\{') + subgraph_pattern = re.compile(r'/subgraph\s+@([\w+]{1,1000)(\([^\)]{1,100}\))?\s+\S[^\{]\{/+') matches = list(subgraph_pattern.finditer(text)) end_pos = 0 for match in matches: @@ -203,11 +191,6 @@ class Parser: subgraph_info.end = end_pos logging.info('Parsed subgraph: %s', subgraph_name) - def count_nodes(self) -> Tuple[int, int]: - total_nodes = len(self.nodes) - total_cnodes = sum(1 for node in self.nodes.values() if node.name.startswith('CNode')) - return total_nodes, total_cnodes - def create_backward_map(self): for node in self.nodes.values(): if node.scope and node.scope.startswith("Gradients"): diff --git a/debug/accuracy_tools/msprobe/mindspore/common/const.py b/debug/accuracy_tools/msprobe/mindspore/common/const.py index 067e783842f13899feaba00476777ded707e9eb7..b457994e99a080fbb02071eb29900e4117b8d44f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/const.py @@ -15,6 +15,7 @@ import numpy as np import mindspore as ms +from mindspore import dtype as mstype from msprobe.core.common.const import Const as CoreConst @@ -23,14 +24,20 @@ class Const: CELL = "cell" API = "api" KERNEL = "kernel" + CELL_AND_API = 'cell_and_api' TOOL_LEVEL_DICT = { CoreConst.LEVEL_L0: CELL, CoreConst.LEVEL_L1: API, - CoreConst.LEVEL_L2: KERNEL + CoreConst.LEVEL_L2: KERNEL, + CoreConst.LEVEL_MIX: CELL_AND_API } + PYNATIVE_MODE = "pynative" + GRAPH_MODE = "graph" GRAPH_GE_MODE = "graph_ge" GRAPH_KBYK_MODE = "graph_kbyk" + PYNATIVE_GRAPH_MODE = 'pynative_graph' + JIT_LEVEL = "jit_level" JIT_LEVEL_O0 = "O0" JIT_LEVEL_O1 = "O1" @@ -61,6 +68,7 @@ class Const: DROPOUT_API_NAME_PREFIX = "dropout" GRAPH_DATA_MODE_LIST = [CoreConst.ALL, CoreConst.INPUT, CoreConst.OUTPUT] + GRAPH_CELL_DUMP_DATA_MODE_LIST = [CoreConst.ALL, CoreConst.FORWARD, CoreConst.BACKWARD] HOOK_MS_PREFIX_DICT = { OPS_DATA_PREFIX: OPS_PREFIX, @@ -69,6 +77,13 @@ class Const: MINT_NN_FUNC_DATA_PREFIX: MINT_NN_FUNC_PREFIX } + NonDifferentiableType = ( + mstype.bool_, mstype.int8, mstype.byte, mstype.uint8, mstype.ubyte, + mstype.int16, mstype.short, mstype.uint16, mstype.ushort, + mstype.int32, mstype.intc, mstype.uint32, mstype.uintc, + mstype.int64, mstype.intp, mstype.uint64, mstype.uintp + ) + class MsCompareConst: # api_info field @@ -88,14 +103,11 @@ class MsCompareConst: MINDTORCH_NPU = "NPU" MINDTORCH_DIST = "Distributed" - - MT_VALID_API_TYPES = [ MINDTORCH, MINDTORCH_FUNC, MINDTORCH_TENSOR ] SUPPORTED_FUSION_LIST = ["flash_attention_score"] - TASK_FIELD = "task" STATISTICS_TASK = "statistics" FRAMEWORK = "framework" @@ -129,8 +141,6 @@ class MsCompareConst: EXCEPTION_SKIP = "exception_skip" - - class FreeBenchmarkConst: ADD_NOISE = "add_noise" BIT_NOISE = "bit_noise" diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index 625842da589a3090cddc75c50175ac577f1777b6..7d491e9d97d8d142668e8f7c326d09ef1a7f3d6c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -13,19 +13,34 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import os import random +import types import mindspore as ms - from mindspore import ops +from mindspore.common.jit_config import JitConfig from mindspore.mint import nn +from msprobe.core.common.const import Const +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.file_utils import path_len_exceeds_limit, check_path_exists, save_npy from msprobe.core.common.log import logger -from msprobe.core.common.const import Const -from msprobe.core.common.utils import CompareException, check_seed_all +from msprobe.core.common.utils import CompareException, check_seed_all, is_save_variable_valid +from msprobe.mindspore.common.const import Const as MsConst + +try: + from mindspore._c_expression import _set_init_iter +except ImportError: + enable_dynamic_kbyk_dump = False +else: + enable_dynamic_kbyk_dump = True + +mindtorch_check_result = None +register_backward_hook_functions = {} +kwargs_exist_in_forward_hook = None class MsprobeStep(ms.train.Callback): @@ -33,6 +48,11 @@ class MsprobeStep(ms.train.Callback): super(MsprobeStep, self).__init__() self.debugger = debugger + def on_train_begin(self, run_context): + self.debugger.start() + if enable_dynamic_kbyk_dump: + _set_init_iter(0) + def on_train_step_begin(self, run_context): self.debugger.start() @@ -82,8 +102,8 @@ def convert_to_int(value): def clean_input_kwargs(cell): - if hasattr(cell, 'input_kwargs'): - del cell.input_kwargs + if hasattr(cell, 'msprobe_input_kwargs'): + del cell.msprobe_input_kwargs def list_lowest_level_directories(root_dir): @@ -152,9 +172,6 @@ def remove_dropout(): nn.functional.dropout = dropout_ext -mindtorch_check_result = None - - def is_mindtorch(): global mindtorch_check_result if mindtorch_check_result is None: @@ -169,11 +186,11 @@ def is_mindtorch(): return mindtorch_check_result -register_backward_hook_functions = {} - - def set_register_backward_hook_functions(): global register_backward_hook_functions + if register_backward_hook_functions: + return + if is_mindtorch(): import torch from msprobe.mindspore.mindtorch import (_call_impl, @@ -192,9 +209,11 @@ def set_register_backward_hook_functions(): def check_save_param(variable, name, save_backward): # try catch this api to skip invalid call - if not isinstance(variable, (list, dict, tuple, ms.Tensor, int, float, str)): + valid_data_types = (ms.Tensor, int, float, str) + if not is_save_variable_valid(variable, valid_data_types): + valid_data_types_with_nested_types = valid_data_types + (dict, tuple, list) logger.warning("PrecisionDebugger.save variable type not valid, " - "should be one of list, dict, tuple, ms.Tensor, int, float or string. " + f"should be one of {valid_data_types_with_nested_types}" "Skip current save process.") raise ValueError if not isinstance(name, str): @@ -207,3 +226,102 @@ def check_save_param(variable, name, save_backward): "should be bool. " "Skip current save process.") raise ValueError + + +def is_graph_mode_cell_dump_allowed(config): + if config.task not in [Const.TENSOR] or is_mindtorch() or not hasattr(ops, 'DumpGradient'): + return False + valid_mix_level = [MsConst.CELL_AND_API, Const.LEVEL_MIX] + if config.level in valid_mix_level and config.execution_mode == MsConst.PYNATIVE_MODE: + return True + return config.level == MsConst.CELL or config.level == Const.LEVEL_L0 + + +@recursion_depth_decorator('msprobe.mindspore.common.utils.is_decorated_by_jit') +def is_decorated_by_jit(func): + closure = getattr(func, '__closure__', []) + if closure: + for obj in closure: + if isinstance(obj.cell_contents, JitConfig): + return True + elif isinstance(obj.cell_contents, types.FunctionType) and hasattr(obj.cell_contents, '__closure__'): + if is_decorated_by_jit(obj.cell_contents): + return True + return False + + +@recursion_depth_decorator('msprobe.mindspore.common.utils.get_cells_and_names') +def get_cells_and_names(model, cells_set=None, name_prefix=''): + cells_set = cells_set if cells_set else set() + if model in cells_set: + return + + cells_set.add(model) + jit_decorated = is_decorated_by_jit(model.construct) + yield name_prefix, model, jit_decorated + if jit_decorated: + return + + children_cells = getattr(model, '_cells') + for name, cell in children_cells.items(): + if cell: + cells_name_prefix = f'{name_prefix}{Const.SEP}{name}' if name_prefix else name + jit_decorated = is_decorated_by_jit(model.construct) + if jit_decorated: + yield cells_name_prefix, cell, jit_decorated + else: + for ele in get_cells_and_names(cell, cells_set, cells_name_prefix): + yield ele + + +def get_cells_and_names_with_index(models): + cells_with_index_in_pynative_mode = {} + cells_with_index_in_graph_mode = {} + + def distinguish_cells(cells): + cells_in_pynative_mode = [] + cells_in_graph_mode = [] + for name, cell, jit_decorated in cells: + if jit_decorated: + cells_in_graph_mode.append((name, cell)) + else: + cells_in_pynative_mode.append((name, cell)) + return cells_in_pynative_mode, cells_in_graph_mode + + if is_mindtorch(): + if isinstance(models, (list, tuple)): + for index, model in enumerate(models): + cells_with_index_in_pynative_mode[str(index)] = model.named_modules() + else: + cells_with_index_in_pynative_mode["-1"] = models.named_modules() + else: + if isinstance(models, (list, tuple)): + for index, model in enumerate(models): + cells = get_cells_and_names(model) + cells_in_pynative_mode, cells_in_graph_mode = distinguish_cells(cells) + cells_with_index_in_pynative_mode[str(index)] = cells_in_pynative_mode + cells_with_index_in_graph_mode[str(index)] = cells_in_graph_mode + else: + cells = get_cells_and_names(models) + cells_in_pynative_mode, cells_in_graph_mode = distinguish_cells(cells) + cells_with_index_in_pynative_mode["-1"] = cells_in_pynative_mode + cells_with_index_in_graph_mode["-1"] = cells_in_graph_mode + + return cells_with_index_in_pynative_mode, cells_with_index_in_graph_mode + + +def has_kwargs_in_forward_hook(): + global kwargs_exist_in_forward_hook + + if kwargs_exist_in_forward_hook is None: + if is_mindtorch(): + kwargs_exist_in_forward_hook = True + return kwargs_exist_in_forward_hook + + try: + func_params = inspect.signature(nn.Cell.register_forward_hook).parameters + kwargs_exist_in_forward_hook = 'with_kwargs' in func_params + except Exception: + kwargs_exist_in_forward_hook = False + + return kwargs_exist_in_forward_hook diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 46f825330dbb8b7ff5ce9d42cef5c6b74e3846f2..5064bedcdb8d65aa4406b77e5e8ae46696faf4d7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -13,41 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from msprobe.core.common.utils import CompareException from msprobe.core.common.file_utils import create_directory from msprobe.core.common.exceptions import FileCheckException from msprobe.mindspore.common.log import logger from msprobe.mindspore.compare.ms_compare import ms_compare -from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json +from msprobe.core.compare.utils import compare_distributed_inner from msprobe.mindspore.compare.ms_graph_compare import GraphMSComparator def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): - if kwargs.get('suffix'): - logger.error("Argument 'suffix' is not supported for compare_distributed.") - raise CompareException(CompareException.INVALID_PARAM_ERROR) - is_print_compare_log = kwargs.get('is_print_compare_log', True) - # get the ranks and match by order - npu_ranks = sorted(check_and_return_dir_contents(npu_dump_dir, 'rank')) - bench_ranks = sorted(check_and_return_dir_contents(bench_dump_dir, 'rank')) - if len(npu_ranks) != len(bench_ranks): - logger.error('The number of ranks in the two runs are different. ' - 'Unable to match the ranks. Please use another folder to compare ' - 'or use compare() api and manually match the ranks.') - raise CompareException(CompareException.INVALID_PATH_ERROR) - for nr, br in zip(npu_ranks, bench_ranks): - npu_data_dir = os.path.join(npu_dump_dir, nr) - bench_data_dir = os.path.join(bench_dump_dir, br) - npu_path = extract_json(npu_data_dir, stack_json=False) - bench_path = extract_json(bench_data_dir, stack_json=False) - - dump_result_param = { - 'npu_json_path': npu_path, - 'bench_json_path': bench_path, - 'is_print_compare_log': is_print_compare_log - } - ms_compare(input_param=dump_result_param, output_path=output_path, suffix=f'_{nr}-{br}', **kwargs) + compare_distributed_inner(npu_dump_dir, bench_dump_dir, output_path, ms_compare, **kwargs) def ms_graph_compare(inputs, outputs): diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 4f158512bb4671d6cb46f706d8bf32772a4c971e..42d973a0e896dc5ee700e17f435275969eee1025 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -13,418 +13,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os -import re -from collections import defaultdict - -import numpy as np -import pandas as pd - -from msprobe.core.common.const import CompareConst, Const -from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import create_directory, load_json, load_npy, load_yaml -from msprobe.core.common.log import logger -from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, \ - check_op_str_pattern_valid, get_dump_mode, set_dump_path, detect_framework_by_dump_json -from msprobe.core.compare.acc_compare import Comparator, ModeConfig -from msprobe.core.compare.check import dtype_mapping +from msprobe.core.compare.acc_compare import Comparator, ModeConfig, MappingConfig, setup_comparison from msprobe.core.compare.layer_mapping import generate_data_mapping_by_layer_mapping -from msprobe.core.compare.utils import set_stack_json_path, reorder_op_x_list - - -class MappingConfig: - def __init__(self, cell_mapping=None, api_mapping=None, data_mapping=None): - self.cell_mapping = cell_mapping - self.api_mapping = api_mapping - self.data_mapping = data_mapping - - -class MSComparator(Comparator): - """ - 用于mindspore动态图同框架/跨框架精度比对，支持md5/summary/all模式。 - cell_mapping: mindspore在cell级别(L0)dump数据和pytorch的module之间的映射关系； - api_mapping: mindspore在api级别(L1)dump数据和pytorch的api之间的映射关系； - data_mapping: mindspore的cell或api的入参/出参和pytorch之间的映射关系； - is_cross_framework: 是否跨框架。 - """ - def __init__(self, mode_config, mapping_config=None, is_cross_framework=False): - super().__init__(mode_config) - self.frame_name = MSComparator.__name__ - - self.stack_mode = mode_config.stack_mode - self.auto_analyze = mode_config.auto_analyze - self.fuzzy_match = mode_config.fuzzy_match - self.dump_mode = mode_config.dump_mode - - if mapping_config: - self.cell_mapping = mapping_config.cell_mapping - self.api_mapping = mapping_config.api_mapping - self.data_mapping = mapping_config.data_mapping - - if self.data_mapping: - self.cross_frame = is_cross_framework - else: - self.cross_frame = self.cell_mapping is not None or self.api_mapping is not None - self.cell_mapping_dict = self.load_mapping_file(self.cell_mapping) - self.api_mapping_dict = self.load_mapping_file(self.api_mapping) - if self.api_mapping is not None: - self.ms_to_pt_mapping = self.load_internal_api() - - if isinstance(self.data_mapping, str) or self.data_mapping is None: - self.data_mapping_dict = self.load_mapping_file(self.data_mapping) - elif isinstance(self.data_mapping, dict): - self.data_mapping_dict = self.data_mapping - else: - raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " - f"{type(self.data_mapping)}") - - @staticmethod - def process_data_name(result): - result['data_name_x'] = result.apply(lambda row: [row['data_name_x'], row['data_name_y']], axis=1) - return result - - def calc_accuracy(self, result_df, header): - condition_no_bench = result_df[CompareConst.BENCH_NAME] == CompareConst.N_A - result_df[condition_no_bench] = result_df[condition_no_bench].fillna(CompareConst.N_A) - result_df.loc[condition_no_bench, CompareConst.ERROR_MESSAGE] = CompareConst.NO_BENCH - - def calc_summary_diff(data_type: str): - def type_check(val): - check_series = pd.Series(False, index=val.index) - val_str = val.astype(str) - check_series[pd.to_numeric(val_str, errors='coerce').notna() | val_str.str.lower().eq('nan')] = True - return check_series - - def get_number(val): - return pd.to_numeric(val.astype(str), errors='coerce') - - ms_val = result_df['NPU ' + data_type] - pt_val = result_df['Bench ' + data_type] - diff_name = data_type.capitalize() + ' diff' - rel_err_name = ('norm' if data_type == 'l2norm' else data_type).capitalize() + 'RelativeErr' - condition_na = ~type_check(ms_val) | ~type_check(pt_val) - result_df.loc[condition_na, [diff_name, rel_err_name]] = CompareConst.N_A - result_df.loc[~(condition_no_bench | condition_na), diff_name] = get_number(ms_val) - get_number(pt_val) - condition_nan_diff = ~condition_no_bench & ~condition_na & result_df[diff_name].isna() - condition_not_nan_diff = ~condition_no_bench & ~condition_na & result_df[diff_name].notna() - result_df.loc[condition_nan_diff, [diff_name, rel_err_name]] = CompareConst.NAN - condition_pt_zero = pt_val == 0 - result_df.loc[condition_not_nan_diff & condition_pt_zero, rel_err_name] = CompareConst.NAN - condition_ref_err = condition_not_nan_diff & ~condition_pt_zero - result_df.loc[condition_ref_err, rel_err_name] = (result_df.loc[condition_ref_err, diff_name] / - pt_val[condition_ref_err] * 100) - result_df.loc[condition_ref_err, rel_err_name] = (result_df.loc[condition_ref_err, rel_err_name] - .abs().astype(str) + '%') - magnitude = get_number(result_df[diff_name]).abs() / ( - pd.Series(np.maximum(get_number(ms_val), get_number(pt_val))).abs() + CompareConst.EPSILON) - return magnitude > CompareConst.MAGNITUDE - - if self.dump_mode == Const.MD5: - condition_md5_equal = result_df[CompareConst.NPU_MD5] == result_df[CompareConst.BENCH_MD5] - result_df.loc[condition_md5_equal, CompareConst.RESULT] = CompareConst.PASS - result_df.loc[~condition_md5_equal & ~condition_no_bench, CompareConst.RESULT] = CompareConst.DIFF - elif self.dump_mode == Const.SUMMARY: - warning_list = [calc_summary_diff(data_type) for data_type in ['max', 'min', 'mean', 'l2norm']] - warning_flag = pd.DataFrame(warning_list).all() - result_df.loc[~condition_no_bench, [CompareConst.RESULT, CompareConst.ERROR_MESSAGE]] = '' - result_df.loc[warning_flag, CompareConst.RESULT] = CompareConst.WARNING - result_df.loc[warning_flag, CompareConst.ERROR_MESSAGE] = 'Need double check api accuracy.' - else: - fill_cols = [CompareConst.COSINE, CompareConst.EUC_DIST, - CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR, - CompareConst.ONE_THOUSANDTH_ERR_RATIO, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO, - CompareConst.ERROR_MESSAGE] - result_df.loc[~condition_no_bench, fill_cols] = '' - result_df.loc[~condition_no_bench, CompareConst.ACCURACY] = CompareConst.ACCURACY_CHECK_YES - return result_df[header] - - def make_result_df(self, result): - header = CompareConst.HEAD_OF_COMPARE_MODE[self.dump_mode][:] - - if self.stack_mode: - header.append(CompareConst.STACK) - if self.dump_mode == Const.ALL: - header.append(CompareConst.DATA_NAME) - result = self.process_data_name(result) - - result.rename(columns={'op_name_x': CompareConst.NPU_NAME, - 'op_name_y': CompareConst.BENCH_NAME, - 'dtype_x': CompareConst.NPU_DTYPE, - 'dtype_y': CompareConst.BENCH_DTYPE, - 'shape_x': CompareConst.NPU_SHAPE, - 'shape_y': CompareConst.BENCH_SHAPE, - 'md5_x': CompareConst.NPU_MD5, - 'md5_y': CompareConst.BENCH_MD5, - 'data_name_x': CompareConst.DATA_NAME, - 'stack_info_x': CompareConst.STACK}, inplace=True) - - npu_summary = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM] - bench_summary = [CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN, - CompareConst.BENCH_NORM] - - def set_summary(summary): - if summary == CompareConst.N_A: - return [CompareConst.N_A] * 4 - summary_list = [] - for i in summary: - if i is None: - summary_list.append(CompareConst.N_A) - elif str(i).lower() == 'nan': - summary_list.append(CompareConst.NAN) - else: - summary_list.append(i) - return summary_list - - result[npu_summary] = result['summary_x'].apply(set_summary).tolist() - result[bench_summary] = result['summary_y'].apply(set_summary).tolist() - - result_df = pd.DataFrame(columns=header) - for h in header: - if h in result.columns: - result_df[h] = result[h] - return self.calc_accuracy(result_df, header) - - def load_internal_api(self): - cur_path = os.path.dirname(os.path.realpath(__file__)) - yaml_path = os.path.abspath(os.path.join(cur_path, CompareConst.INTERNAL_API_MAPPING_FILE)) - return load_yaml(yaml_path) - - def load_mapping_file(self, mapping_file): - if isinstance(mapping_file, str): - mapping_dict = load_yaml(mapping_file) - else: - mapping_dict = {} - return mapping_dict - - def process_cell_mapping(self, npu_op_name): - if not npu_op_name: - return CompareConst.N_A - param_grad_flag = Const.PARAMS_GRAD in npu_op_name.split(Const.SEP) - if not param_grad_flag and not re.search(Const.REGEX_FORWARD_BACKWARD, npu_op_name): - return CompareConst.N_A - npu_op_name = npu_op_name.replace("Cell", "Module", 1) - if self.cell_mapping_dict: - # get cell name & class name from op_name - # Cell.fc1.Dense.forward.0.input.0 - cell_name = re.split(r'\.(?:forward|backward|parameters_grad)\.', npu_op_name.split(Const.SEP, 1)[-1])[0] - if cell_name in self.cell_mapping_dict: - npu_op_name = npu_op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) - return npu_op_name - - def read_npy_data(self, dir_path, file_name, load_pt_file=False): - if not file_name: - return None - data_path = os.path.join(dir_path, file_name) - if load_pt_file: - import torch - from msprobe.pytorch.common.utils import load_pt - data_value = load_pt(data_path, True).detach() - if data_value.dtype == torch.bfloat16: - data_value = data_value.to(torch.float32) - data_value = data_value.numpy() - else: - data_value = load_npy(data_path) - return data_value - - def process_internal_api_mapping(self, npu_op_name): - # get api name & class name from op_name - # Functional.addcmul.0.forward.input.0 - ms_api_name = self.get_api_name(npu_op_name.split(Const.SEP)) - class_name = ms_api_name.split(Const.SEP)[0] - if class_name == "Mint": - return npu_op_name.replace("Mint", "Torch") - elif class_name == "MintFunctional": - return npu_op_name.replace("MintFunctional", "Functional") - elif self.ms_to_pt_mapping.get(ms_api_name): - return npu_op_name.replace(ms_api_name, self.ms_to_pt_mapping.get(ms_api_name)) - else: - return npu_op_name - - def get_api_name(self, api_list): - try: - api_name = api_list[0] + Const.SEP + api_list[1] - except IndexError as error: - logger.error(f'Failed to retrieve API name, please check if the dump data is reasonable') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from error - return api_name - - def compare_process(self, file_lists): - npu_json_path, bench_json_path, stack_json_path = file_lists - npu_json_data = load_json(npu_json_path) - bench_json_data = load_json(bench_json_path) - stack_json_data = load_json(stack_json_path) if self.stack_mode else None - - npu_df = self.gen_data_df(npu_json_data, stack_json_data) - bench_df = self.gen_data_df(bench_json_data, stack_json_data) - if self.cell_mapping: - npu_df[CompareConst.COMPARE_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_cell_mapping) - elif self.api_mapping: - npu_df[CompareConst.COMPARE_KEY] = npu_df[CompareConst.OP_NAME].apply(self.process_internal_api_mapping) - if isinstance(self.api_mapping, str): - self.modify_compare_data_with_user_mapping(npu_df, bench_df) - else: - npu_df[CompareConst.COMPARE_KEY] = npu_df[CompareConst.OP_NAME] - npu_df[[Const.DTYPE, Const.SHAPE]] = npu_df[[Const.DTYPE, Const.SHAPE]].astype(str) - bench_df[[Const.DTYPE, Const.SHAPE]] = bench_df[[Const.DTYPE, Const.SHAPE]].astype(str) - npu_df[CompareConst.COMPARE_SHAPE] = npu_df[Const.SHAPE] - bench_df[CompareConst.COMPARE_KEY] = bench_df[CompareConst.OP_NAME] - bench_df[CompareConst.COMPARE_SHAPE] = bench_df[Const.SHAPE] - match_result = pd.merge(npu_df, bench_df, on=[CompareConst.COMPARE_KEY, CompareConst.COMPARE_SHAPE], - how='outer') - match_result = match_result[match_result['op_name_x'].notna()].fillna(CompareConst.N_A) - - def gen_dtype_condition(): - npu_dtype = match_result['dtype_x'] - bench_dtype = match_result['dtype_y'] - if self.cross_frame: - npu_dtype = npu_dtype.map(dtype_mapping).fillna(npu_dtype) - - equal_condition = npu_dtype == bench_dtype - match_condition = ( - (npu_dtype.isin(CompareConst.DTYPE_MATCH_GROUPS[0]) & bench_dtype.isin( - CompareConst.DTYPE_MATCH_GROUPS[0])) | - (npu_dtype.isin(CompareConst.DTYPE_MATCH_GROUPS[1]) & bench_dtype.isin( - CompareConst.DTYPE_MATCH_GROUPS[1])) - ) - return equal_condition | match_condition - - match_result.loc[~gen_dtype_condition(), [i + '_y' for i in bench_df.columns]] = CompareConst.N_A - return self.make_result_df(match_result) - - def modify_compare_data_with_user_mapping(self, npu_df, bench_df): - def get_api_indices_dict(op_name_df): - api_indices_dict = defaultdict(list) - for op_index, name in enumerate(op_name_df[CompareConst.OP_NAME]): - api = self.get_api_name(name.split(Const.SEP)) - api_indices_dict[api].append(op_index) - return api_indices_dict - - ms_api_indices_dict = get_api_indices_dict(npu_df) - pt_api_indices_dict = get_api_indices_dict(bench_df) - - def gen_input_compare_key(pattern, term): - flag = True - for i, prefix in enumerate(mapping_dict.get(f'ms_{term}')): - if op_name.split(pattern)[1].startswith(str(prefix)): - npu_df.loc[index, CompareConst.COMPARE_KEY] = ( - op_name.replace(pattern + str(prefix), - pattern + str(mapping_dict.get(f'pt_{term}')[i]))) - flag = False - return flag - - for mapping_dict in self.api_mapping_dict: - keys_to_compare = [ - ('ms_args', 'pt_args'), - ('ms_output', 'pt_output'), - ('ms_parameters', 'pt_parameters'), - ('ms_parameters_grad', 'pt_parameters_grad'), - ] - if not all(len(mapping_dict.get(k1, [])) == len(mapping_dict.get(k2, [])) for k1, k2 in keys_to_compare): - logger.warning('The user-defined mapping table is incorrect,\ - make sure that the number of parameters is equal') - continue - - ms_api, pt_api = mapping_dict.get('ms_api'), mapping_dict.get('pt_api') - if ms_api not in ms_api_indices_dict or pt_api not in pt_api_indices_dict: - continue - for index in ms_api_indices_dict.get(ms_api): - op_name = npu_df.loc[index, CompareConst.OP_NAME].replace(ms_api, pt_api, 1) - if CompareConst.INPUT_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.INPUT_PATTERN, 'args') - elif CompareConst.KWARGS_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.KWARGS_PATTERN, 'args') - elif CompareConst.OUTPUT_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.OUTPUT_PATTERN, 'output') - elif CompareConst.PARAMS_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.PARAMS_PATTERN, 'parameters') - elif CompareConst.PARAMS_GRAD_PATTERN in op_name: - is_abandoned = gen_input_compare_key(CompareConst.PARAMS_GRAD_PATTERN, 'parameters_grad') - else: - logger.error(f'Excepted op_name: {op_name}') - raise CompareException(CompareException.INVALID_DATA_ERROR) - if is_abandoned: - npu_df.loc[index, CompareConst.COMPARE_KEY] = op_name + 'abandoned' - - def gen_data_df(self, data_json, stack_json_data): - result = { - CompareConst.OP_NAME: [], - Const.DTYPE: [], - Const.SHAPE: [], - Const.SUMMARY: [], - 'stack_info': [] - } - if self.dump_mode == Const.ALL: - result['data_name'] = [] - elif self.dump_mode == Const.MD5: - result[Const.MD5] = [] - for data_name in data_json['data']: - check_op_str_pattern_valid(data_name) - merge_list = self.gen_merge_list(data_json, data_name, stack_json_data) - if not merge_list: - continue - - op_name_list = merge_list.get(CompareConst.OP_NAME) - summary_list = merge_list.get(Const.SUMMARY) - data_name_list = merge_list.get('data_name') - op_name_reorder, summary_reorder, data_name_reorder = reorder_op_x_list(op_name_list, - summary_list, - data_name_list) - for op_name in op_name_reorder: - result[CompareConst.OP_NAME].append(op_name) - if (CompareConst.INPUT_PATTERN in op_name) or (CompareConst.KWARGS_PATTERN in op_name): - struct = merge_list[CompareConst.INPUT_STRUCT].pop(0) - elif CompareConst.OUTPUT_PATTERN in op_name: - struct = merge_list[CompareConst.OUTPUT_STRUCT].pop(0) - elif CompareConst.PARAMS_PATTERN in op_name: - struct = merge_list[CompareConst.PARAMS_STRUCT].pop(0) - else: - struct = merge_list[CompareConst.PARAMS_GRAD_STRUCT].pop(0) - result[Const.DTYPE].append(struct[0]) - result[Const.SHAPE].append(struct[1]) - if self.dump_mode == Const.MD5: - result[Const.MD5].append(struct[2]) - result[Const.SUMMARY].append(summary_reorder.pop(0)) - result['stack_info'].append(merge_list['stack_info'][0] if self.stack_mode else None) - if self.dump_mode == Const.ALL: - result['data_name'].append(data_name_reorder.pop(0)) - return pd.DataFrame(result) +from msprobe.mindspore.compare.utils import read_npy_data, check_cross_framework -def check_cross_framework(bench_json_path): - framework = detect_framework_by_dump_json(bench_json_path) - if framework == Const.PT_FRAMEWORK: - return True +def read_real_data(npu_dir, npu_data_name, bench_dir, bench_data_name, cross_frame) -> tuple: + n_value = read_npy_data(npu_dir, npu_data_name) + if cross_frame: + from msprobe.pytorch.compare.utils import read_pt_data + b_value = read_pt_data(bench_dir, bench_data_name) else: - return False + b_value = read_npy_data(bench_dir, bench_data_name) + return n_value, b_value def ms_compare(input_param, output_path, **kwargs): - try: - auto_analyze = kwargs.get('auto_analyze', True) - fuzzy_match = kwargs.get('fuzzy_match', False) - cell_mapping = kwargs.get('cell_mapping', None) - api_mapping = kwargs.get('api_mapping', None) - data_mapping = kwargs.get('data_mapping', None) - layer_mapping = kwargs.get('layer_mapping', None) - suffix = kwargs.get('suffix', '') + config = setup_comparison(input_param, output_path, **kwargs) - set_dump_path(input_param) - dump_mode = get_dump_mode(input_param) - if 'stack_json_path' in input_param: - stack_mode = kwargs.get('stack_mode', False) - else: - stack_mode = set_stack_json_path(input_param) # set stack_mode and set "stack_json_path" in input_param - check_configuration_param(stack_mode, auto_analyze, fuzzy_match, input_param.get('is_print_compare_log', True)) - create_directory(output_path) - check_compare_param(input_param, output_path, dump_mode, stack_mode) - except (CompareException, FileCheckException) as error: - logger.error('Compare failed. Please check the arguments and do it again!') - raise CompareException(error.code) from error - if layer_mapping: - data_mapping = generate_data_mapping_by_layer_mapping(input_param, layer_mapping, output_path) + if config.layer_mapping: + config.data_mapping = generate_data_mapping_by_layer_mapping(input_param, config.layer_mapping, output_path) - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(cell_mapping, api_mapping, data_mapping) is_cross_framework = check_cross_framework(input_param.get('bench_json_path')) - ms_comparator = MSComparator(mode_config, mapping_config, is_cross_framework) - ms_comparator.compare_core(input_param, output_path, suffix=suffix) + mode_config = ModeConfig(config.stack_mode, config.auto_analyze, config.fuzzy_match, + config.dump_mode, config.compared_file_type) + mapping_config = MappingConfig(config.cell_mapping, config.api_mapping, config.data_mapping) + ms_comparator = Comparator(read_real_data, mode_config, mapping_config, is_cross_framework) + ms_comparator.compare_core(input_param, output_path, suffix=config.suffix) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/utils.py b/debug/accuracy_tools/msprobe/mindspore/compare/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7a9c78e8f74426c23982723fcf90f729fc9e694c --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/utils.py @@ -0,0 +1,37 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from msprobe.core.common.const import Const +from msprobe.core.common.file_utils import load_npy, FileChecker, FileCheckConst +from msprobe.core.common.utils import detect_framework_by_dump_json + + +def read_npy_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.NUMPY_SUFFIX, False) + data_path = path_checker.common_check() + data_value = load_npy(data_path) + return data_value + + +def check_cross_framework(bench_json_path): + framework = detect_framework_by_dump_json(bench_json_path) + return framework == Const.PT_FRAMEWORK diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 92155b4ec4ebd636477ef67f1c75b43e7a82b802..1862419bb68250b6b18d6814ddefa1f2f140ebb6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -41,8 +41,12 @@ class DebuggerConfig: self.check_mode = task_config.check_mode self.framework = Const.MS_FRAMEWORK self.summary_mode = task_config.summary_mode + self.stat_cal_mode = task_config.stat_cal_mode if hasattr(task_config, 'stat_cal_mode') else None + self.device_stat_precision_mode = task_config.device_stat_precision_mode \ + if hasattr(task_config, 'device_stat_precision_mode') else None self.async_dump = common_config.async_dump if common_config.async_dump else False self.check() + self._check_statistics_config(task_config) create_directory(self.dump_path) if self.task == Const.FREE_BENCHMARK: @@ -53,11 +57,13 @@ class DebuggerConfig: self.stage = FreeBenchmarkConst.DEFAULT_STAGE if not task_config.fuzz_stage else task_config.fuzz_stage if self.handler_type == FreeBenchmarkConst.FIX and \ self.pert_type != FreeBenchmarkConst.DEFAULT_PERT_TYPE: - raise ValueError("pert_mode must be improve_precision or empty when handler_type is fix, " - f"but got {self.pert_type}.") + logger.error("pert_mode must be improve_precision or empty when handler_type is fix, " + f"but got {self.pert_type}.") + raise ValueError if self.stage == Const.BACKWARD and self.handler_type == FreeBenchmarkConst.FIX: - raise ValueError("handler_type must be check or empty when fuzz_stage is backward, " - f"but got {self.handler_type}.") + logger.error("handler_type must be check or empty when fuzz_stage is backward, " + f"but got {self.handler_type}.") + raise ValueError self.dump_level = FreeBenchmarkConst.DEFAULT_DUMP_LEVEL def check(self): @@ -74,8 +80,12 @@ class DebuggerConfig: self.check_mode = "all" if not isinstance(self.async_dump, bool): raise Exception("The parameters async_dump should be bool.") - if self.async_dump and self.task == Const.TENSOR and not self.list: - raise Exception("The parameters async_dump is true in tensor task, the parameters list cannot be empty.") + if self.async_dump and self.task == Const.TENSOR: + if self.level_ori == Const.LEVEL_DEBUG: + self.list = [] # async_dump + debug level case ignore list + if not self.list and self.level_ori != Const.LEVEL_DEBUG: + raise Exception("The parameters async_dump is true in tensor task," + " the parameters list cannot be empty.") if self.task == Const.STRUCTURE and self.level_ori not in [Const.LEVEL_L0, Const.LEVEL_MIX]: logger.warning_on_rank_0( f"When the task is set to structure, the level should be one of {[Const.LEVEL_L0, Const.LEVEL_MIX]}. " @@ -96,3 +106,14 @@ class DebuggerConfig: if not self.list or len(self.list) != 1: raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, f"When level is set to L2, the list must be configured as a list with one api name.") + + def _check_statistics_config(self, task_config): + if self.task != Const.STATISTICS: + return + self.tensor_list = [] + if not hasattr(task_config, "tensor_list"): + return + if self.level_ori == Const.LEVEL_DEBUG and task_config.tensor_list: + logger.warning_on_rank_0("When level is set to debug, the tensor_list will be invalid.") + return + self.tensor_list = task_config.tensor_list diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 4f2109504fbc2c54bc73daa42ecd96567ac90502..e6758e2b55cba57014443838ccd2d1a42f52bb66 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -22,10 +22,14 @@ from mindspore._c_expression import MSContext from msprobe.core.common.const import Const, FileCheckConst, MsgConst from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common.file_utils import FileChecker -from msprobe.core.common.utils import get_real_step_or_rank, check_init_step +from msprobe.core.common.utils import get_real_step_or_rank, check_init_step, check_token_range from msprobe.mindspore.cell_processor import CellProcessor from msprobe.mindspore.common.const import Const as MsConst -from msprobe.mindspore.common.utils import set_register_backward_hook_functions, check_save_param +from msprobe.mindspore.common.utils import ( + set_register_backward_hook_functions, + check_save_param, + is_graph_mode_cell_dump_allowed +) from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.hook_cell.api_register import get_api_register from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell @@ -34,6 +38,14 @@ from msprobe.mindspore.ms_config import parse_json_config from msprobe.mindspore.runtime import Runtime from msprobe.mindspore.service import Service from msprobe.mindspore.task_handler_factory import TaskHandlerFactory +from msprobe.mindspore.dump.graph_mode_cell_dump import GraphModeCellDump + +try: + from mindspore._c_expression import _dump_start, _dump_stop, _dump_step, _set_init_iter, _dump_set_dynamic +except ImportError: + enable_dynamic_kbyk_dump = False +else: + enable_dynamic_kbyk_dump = True try: from msprobe.lib import _msprobe_c @@ -85,6 +97,7 @@ class PrecisionDebugger: self.config = DebuggerConfig(common_config, task_config) if self._need_msprobe_c() and _msprobe_c: + os.environ["MS_HOOK_ENABLE"] = "on" _msprobe_c._PrecisionDebugger(framework="MindSpore", config_path=config_path) self.config.execution_mode = self._get_execution_mode() @@ -94,13 +107,15 @@ class PrecisionDebugger: Runtime.step_count = 0 Runtime.is_running = False + if enable_dynamic_kbyk_dump: + _dump_set_dynamic() @staticmethod def check_input_params(args): if args.config_path is not None: if not isinstance(args.config_path, str): raise MsprobeException( - MsprobeException.INVALID_PARAM_ERROR, f"config_path must be a string") + MsprobeException.INVALID_PARAM_ERROR, "config_path must be a string") file_checker = FileChecker( file_path=args.config_path, path_type=FileCheckConst.FILE, file_type=FileCheckConst.JSON_SUFFIX) file_checker.common_check() @@ -112,7 +127,7 @@ class PrecisionDebugger: if args.dump_path is not None: if not isinstance(args.dump_path, str): raise MsprobeException( - MsprobeException.INVALID_PARAM_ERROR, f"dump_path must be a string") + MsprobeException.INVALID_PARAM_ERROR, "dump_path must be a string") if args.level is not None and args.level not in Const.LEVEL_LIST: raise MsprobeException( @@ -137,7 +152,7 @@ class PrecisionDebugger: return MsConst.PYNATIVE_MODE @staticmethod - def _is_graph_dump(config): + def _is_graph_dump(config: DebuggerConfig): if config.level != MsConst.KERNEL: return False if not config.list: @@ -147,7 +162,7 @@ class PrecisionDebugger: return is_graph @classmethod - def start(cls, model=None): + def start(cls, model=None, token_range=None): instance = cls._instance if not instance: raise Exception(MsgConst.NOT_CREATED_INSTANCE) @@ -155,17 +170,24 @@ class PrecisionDebugger: _msprobe_c._PrecisionDebugger().start() if instance.task in PrecisionDebugger.task_not_need_service: return - + check_token_range(token_range) instance.config.execution_mode = cls._get_execution_mode() if cls._need_service(): if not instance.service: instance.service = Service(instance.config) - instance.service.start(model) + instance.service.start(model, token_range) else: if not instance.first_start: get_api_register().restore_all_api() - handler = TaskHandlerFactory.create(instance.config) + handler = TaskHandlerFactory.create(instance.config, model) handler.handle() + if enable_dynamic_kbyk_dump: + _set_init_iter(0) + if enable_dynamic_kbyk_dump: + is_valid_rank = (not instance.config.rank or Runtime.rank_id in instance.config.rank) + is_valid_step = (not instance.config.step or Runtime.step_count in instance.config.step) + if is_valid_rank and is_valid_step: + _dump_start() instance.first_start = True Runtime.is_running = True @@ -180,30 +202,38 @@ class PrecisionDebugger: instance = cls._instance if not instance: raise Exception(MsgConst.NOT_CREATED_INSTANCE) - if cls._need_msprobe_c() and _msprobe_c: - _msprobe_c._PrecisionDebugger().stop() if instance.task == Const.GRAD_PROBE: instance.gm.stop() if instance.task in PrecisionDebugger.task_not_need_service: return if instance.service: instance.service.stop() + if enable_dynamic_kbyk_dump: + _dump_stop() + if cls._need_msprobe_c() and _msprobe_c: + _msprobe_c._PrecisionDebugger().stop() Runtime.is_running = False @classmethod def step(cls): instance = cls._instance + if not instance: raise Exception(MsgConst.NOT_CREATED_INSTANCE) - if cls._need_msprobe_c() and _msprobe_c: - _msprobe_c._PrecisionDebugger().step() if instance.task in PrecisionDebugger.task_not_need_service: return + if instance.service: instance.service.step() + if is_graph_mode_cell_dump_allowed(instance.config): + GraphModeCellDump.step() + if enable_dynamic_kbyk_dump: + _dump_step(1) + if cls._need_msprobe_c() and _msprobe_c: + _msprobe_c._PrecisionDebugger().step() + HOOKCell.cell_count = defaultdict(int) CellProcessor.reset_cell_stats() - Runtime.step_count += 1 @classmethod @@ -240,6 +270,32 @@ class PrecisionDebugger: raise Exception(MsgConst.NOT_CREATED_INSTANCE) check_init_step(step) instance.service.init_step = step + instance.service.loop = 0 + + @classmethod + def register_custom_api(cls, module, api_name, api_prefix=None): + if not api_prefix: + api_prefix = getattr(module, "__name__", "Custom") + if not isinstance(api_prefix, str): + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, "api_prefix must be string") + if not hasattr(module, api_name): + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, f"module {str(module)} does not have {api_name}") + instance = cls._instance + if not instance: + raise Exception(MsgConst.NOT_CREATED_INSTANCE) + instance.service.register_custom_api(module, api_name, api_prefix) + + @classmethod + def restore_custom_api(cls, module, api): + if not hasattr(module, api): + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, f"module {str(module)} does not have {api}") + instance = cls._instance + if not instance: + raise Exception(MsgConst.NOT_CREATED_INSTANCE) + instance.service.restore_custom_api(module, api) @classmethod def _need_service(cls): @@ -250,7 +306,7 @@ class PrecisionDebugger: return False else: return instance.config.task != Const.FREE_BENCHMARK and not instance._is_graph_dump(instance.config) - + @classmethod def _need_msprobe_c(cls): instance = cls._instance diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/cell_dump_process.py b/debug/accuracy_tools/msprobe/mindspore/dump/cell_dump_process.py new file mode 100644 index 0000000000000000000000000000000000000000..722f20af850640e139b7aa4c9c3be31ccae06cbc --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/cell_dump_process.py @@ -0,0 +1,621 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +from multiprocessing import Pool +import os +import re +import time + +import numpy as np +import mindspore as ms +from mindspore import nn, ops + +from msprobe.core.common.const import Const as CoreConst +from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.file_utils import load_npy, save_json, remove_path, load_yaml +from msprobe.mindspore.common.log import logger + +CONSTRUCT_FILE_NAME = "construct.json" +DEFAULT_RANK_DIR = "rank0" +KEY_LAYERS = "layers" +construct = {} +cell_list = [] +KEY_SIDE_EFFECT = "side_effect_io" +KEY_TOPLAYER = "TopLayer" +KEY_FORWARD = CoreConst.FORWARD +KEY_BACKWARD = CoreConst.BACKWARD +KEY_INPUT = CoreConst.INPUT +KEY_OUTPUT = CoreConst.OUTPUT +td = ops.TensorDump() +if (ms.__version__ >= "2.5.0"): + td_in = ops.TensorDump("in") +else: + td_in = ops.TensorDump() +dump_gradient_op_existed = False +if hasattr(ops, 'DumpGradient'): + gd = ops.DumpGradient() + dump_gradient_op_existed = True +else: + logger.warning('The operator "DumpGradient" does not exist. Cell dump can not work in graph mode.') +td.add_prim_attr(KEY_SIDE_EFFECT, False) +td_in.add_prim_attr(KEY_SIDE_EFFECT, False) +np_ms_dtype_dict = { + "bool": ms.bool_, + "int8": ms.int8, + "byte": ms.byte, + "int16": ms.int16, + "short": ms.short, + "int32": ms.int32, + "intc": ms.intc, + "int64": ms.int64, + "intp": ms.intp, + "uint8": ms.uint8, + "ubyte": ms.ubyte, + "uint16": ms.uint16, + "ushort": ms.ushort, + "uint32": ms.uint32, + "uintc": ms.uintc, + "uint64": ms.uint64, + "uintp": ms.uintp, + "float16": ms.float16, + "half": ms.half, + "float32": ms.float32, + "single": ms.single, + "float64": ms.float64, + "double": ms.double, + "bfloat16": ms.bfloat16, + "complex64": ms.complex64, + "complex128": ms.complex128 +} + + +def gen_file_path(dump_path, cell_prefix, suffix, io_type, index): + step_path = os.path.join(dump_path, "{step}") + rank_path = os.path.join(step_path, "{rank}") + data_path = os.path.join(rank_path, CoreConst.DUMP_TENSOR_DATA) + file_name = cell_prefix + CoreConst.SEP + suffix + CoreConst.SEP + io_type + CoreConst.SEP + str(index) + return os.path.join(data_path, file_name) + + +def need_tensordump_in(cell_obj, attr, index): + if not hasattr(cell_obj, attr): + return False + attr_values = getattr(cell_obj, attr) + if index >= len(attr_values): + return False + return attr_values[index] == "in" + + +def cell_construct_wrapper(func, self): + def new_construct(self, *args, **kwargs): + new_args = [] + out_list = [] + + index = 0 + item = None + backward_or_all = self.data_mode in ["backward", "all"] + forward_or_all = self.data_mode in ["forward", "all"] + # The inputs of the cell. + for index, item in enumerate(args): + if backward_or_all and ops.is_tensor(item): + if need_tensordump_in(self, 'input_dump_mode', index): + item = gd(gen_file_path(self.dump_path, self.cell_prefix, KEY_BACKWARD, KEY_OUTPUT, index), + item, "in") + else: + item = gd(gen_file_path(self.dump_path, self.cell_prefix, KEY_BACKWARD, KEY_OUTPUT, index), + item, "out") + if forward_or_all and ops.is_tensor(item): + if need_tensordump_in(self, 'input_dump_mode', index): + temp = td_in( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_INPUT, index), + item + ) + else: + temp = td( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_INPUT, index), + item + ) + item = ops.depend(item, temp) + new_args.append(item) + + out = func(*new_args, **kwargs) + + # The outputs of the cell. + if isinstance(out, tuple): + for index, item in enumerate(out): + if backward_or_all and ops.is_tensor(item): + if need_tensordump_in(self, 'output_dump_mode', index): + item = gd(gen_file_path(self.dump_path, self.cell_prefix, KEY_BACKWARD, KEY_INPUT, index), + item, "in") + else: + item = gd(gen_file_path(self.dump_path, self.cell_prefix, KEY_BACKWARD, KEY_INPUT, index), + item, "out") + if forward_or_all and ops.is_tensor(item): + if need_tensordump_in(self, 'output_dump_mode', index): + temp = td_in( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_OUTPUT, index), + item + ) + else: + temp = td( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_OUTPUT, index), + item + ) + item = ops.depend(item, temp) + out_list.append(item) + elif forward_or_all and not ops.is_tensor(item): + out_list.append(item) + out_list = tuple(out_list) + return out_list + else: + if backward_or_all: + if need_tensordump_in(self, 'output_dump_mode', index): + out = gd(gen_file_path(self.dump_path, self.cell_prefix, KEY_BACKWARD, KEY_INPUT, 0), + out, "in") + else: + out = gd(gen_file_path(self.dump_path, self.cell_prefix, KEY_BACKWARD, KEY_INPUT, 0), + out, "out") + if forward_or_all and ops.is_tensor(out): + if need_tensordump_in(self, 'output_dump_mode', index): + temp = td_in( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_OUTPUT, 0), + out + ) + else: + temp = td( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_OUTPUT, 0), + out + ) + out = ops.depend(out, temp) + return out + + return new_construct.__get__(self, type(self)) + + +# 获取目录下所有文件名并根据TensorDump落盘自增id从小到大排序 +def sort_filenames(path): + filenames = os.listdir(path) + id_pattern = re.compile(rf'{CoreConst.REPLACEMENT_CHARACTER}(\d+){CoreConst.NUMPY_SUFFIX}$') + filenames.sort(key=lambda x: int(id_pattern.findall(x)[0])) + return filenames + + +# 删除重复dump的文件：自定义文件名相同，并且数据相同 +def del_same_file(path, filenames): + result_list = [] + seen_prefixes = {} + for current_filename in filenames: + parts = current_filename.rsplit(CoreConst.REPLACEMENT_CHARACTER, 1) + prefix = parts[0] + if prefix not in seen_prefixes: + result_list.append(current_filename) + seen_prefixes[prefix] = current_filename + else: + current_file_path = os.path.join(path, current_filename) + current_file = load_npy(current_file_path) + prev_filename = seen_prefixes[prefix] + prev_file_path = os.path.join(path, prev_filename) + prev_file = load_npy(prev_file_path) + if np.array_equal(current_file, prev_file): + remove_path(current_file_path) + logger.warning(f"{current_file_path} is deleted!") + else: + result_list.append(current_filename) + return result_list + + +def rename_filename(path): + filenames = sort_filenames(path) + filenames = del_same_file(path, filenames) + + filename_dict = {} + for filename in filenames: + name_field = filename.rsplit(CoreConst.REPLACEMENT_CHARACTER, 1)[0] + + if name_field in filename_dict: + filename_dict[name_field] += 1 + else: + filename_dict[name_field] = 0 + + cell_index = filename_dict[name_field] + + # 修改文件名，增加重复调用Cell的序号 + if CoreConst.FORWARD_PATTERN in filename: + # Format: Cell.{cell_name}.{class_name}.{forward/backward}.{number}.{input/output}.{index}_{dtype}_{id}.npy + new_file_name = filename.replace(CoreConst.FORWARD_PATTERN, + CoreConst.FORWARD_PATTERN + str(cell_index) + CoreConst.SEP) + if CoreConst.BACKWARD_PATTERN in filename: + new_file_name = filename.replace(CoreConst.BACKWARD_PATTERN, + CoreConst.BACKWARD_PATTERN + str(cell_index) + CoreConst.SEP) + os.rename(os.path.join(path, filename), os.path.join(path, new_file_name)) + logger.info("==========The rename_filename phase is Finished!==========") + + +# Extract the field between the first "." and the third to last ".", i.e. {cell_name} +def get_cell_name(str): + parts = str.split(CoreConst.SEP) + if len(parts) < 4: + return None + start_index = 1 + end_index = len(parts) - 3 + return CoreConst.SEP.join(parts[start_index:end_index]) + + +# Extract the field between the last "." and the second to last ".", i.e. {data_made} +def get_data_mode(str): + last_dot_index = str.rfind(CoreConst.SEP) + second_last_dot_index = str.rfind(CoreConst.SEP, 0, last_dot_index) + data_mode = str[second_last_dot_index + 1:last_dot_index] + return data_mode + + +# 判断二者之间是否存在父子关系 +def check_relation(cell_name, parent_cell_name): + layers_pattern = rf"{CoreConst.SEP}{KEY_LAYERS}{CoreConst.SEP}\d+$" + last_dot_index = cell_name.rfind(CoreConst.SEP) + if last_dot_index == -1: + return False + # 如果cell_name最后一个'.'之前的字段等于parent_cell_name，则判定存在父子关系 + sub_cell_name = cell_name[:last_dot_index] + if sub_cell_name == parent_cell_name: + return True + elif re.search(layers_pattern, cell_name): + # 如果cell_name以".layer.{layer_id}"结尾，且去掉该字段后等于parent_cell_name，则判定存在父子关系 + sub_cell_name = re.sub(layers_pattern, '', cell_name) + if sub_cell_name == parent_cell_name: + return True + return False + + +def get_construct(cell_list_input): + for cell in cell_list_input: + cell_name = get_cell_name(cell) + cell_data_mode = get_data_mode(cell) + found_flag = False + for parent_cell in cell_list_input: + parent_cell_name = get_cell_name(parent_cell) + parent_data_mode = get_data_mode(parent_cell) + has_relation = check_relation(cell_name, parent_cell_name) + if has_relation and parent_data_mode == cell_data_mode: + construct.update({cell: parent_cell}) + found_flag = True + break + if not found_flag: + construct.update({cell: None}) + + +def generate_construct(path): + global construct + filenames = sort_filenames(path) + + # 提取文件名中Cell.{cell_name}.{class_name}.{data_mode}.{重复调用此cell的序号}字段，并存入cell_list + for filename in filenames: + point_position = 3 + mid_field = filename.rsplit(CoreConst.SEP, point_position)[0] + if KEY_INPUT in filename: + if mid_field in cell_list: + cell_list.remove(mid_field) + cell_list.append(mid_field) + else: + if mid_field not in cell_list: + index = filenames.index(filename) + output_field = mid_field + KEY_OUTPUT + find_flag = False + for filename_other in cell_list[index + 1:]: + if output_field in filename_other: + find_flag = True + if find_flag is False: + cell_list.append(mid_field) + + get_construct(cell_list) + + # 生成JSON文件 + rank_dir = os.path.dirname(path) + json_path = os.path.join(rank_dir, CONSTRUCT_FILE_NAME) + save_json(json_path, construct, indent=1) + + # 清空'construct'继续处理下一个路径下的数据 + construct = {} + logger.info(f"Construct data saved to {json_path}") + + +def process_file(file_path): + try: + # 读取.npy文件内容 + npy_content = load_npy(file_path) + logger.debug(f"Loaded {file_path}: shape is {npy_content.shape}, dtype is {npy_content.dtype}") + + # 文件名举例:Cell.network._backbone.loss.CrossEntropyLoss.forward.0.input.0_float32_165.npy + parts = os.path.basename(file_path).split(CoreConst.SEP) + data_dtype = "" + # 获取0_float32_165或者0_in_float32_165中的float32 + data_dtype_list = parts[-2].split('_') + if len(data_dtype_list) > 1: + data_dtype = data_dtype_list[-2] + # op_name是Cell.network._backbone.loss.CrossEntropyLoss.forward.0 + op_name = CoreConst.SEP.join(parts[:-3]) + ms_dtype = np_ms_dtype_dict.get(data_dtype) + if ms_dtype is None: + logger.warning(f"Get dtype None from file {file_path}") + + # 修改落盘文件名字，去掉TensorDump自带的数据类型和自增id字段 + data_file_name = os.path.basename(file_path) + data_file_dir = os.path.dirname(file_path) + parts = data_file_name.split(CoreConst.SEP) + if len(parts) >= 2: + param_index = parts[-2].split(CoreConst.REPLACEMENT_CHARACTER)[0] + pre_parts = CoreConst.SEP.join(parts[:-2]) + new_file_name = pre_parts + CoreConst.SEP + param_index + CoreConst.NUMPY_SUFFIX + os.rename(os.path.join(data_file_dir, data_file_name), os.path.join(data_file_dir, new_file_name)) + logger.debug(f"{data_file_name} is renamed to {new_file_name}") + else: + logger.warning(f"Failed to rename {data_file_name}.") + new_file_name = data_file_name + + tensor_json = { + CoreConst.TYPE: 'mindspore.Tensor', + CoreConst.DTYPE: str(ms_dtype), + CoreConst.SHAPE: list(npy_content.shape), + CoreConst.MAX: npy_content.max().item(), + CoreConst.MIN: npy_content.min().item(), + CoreConst.MEAN: npy_content.mean().item(), + CoreConst.NORM: np.linalg.norm(npy_content).item(), + CoreConst.DATA_NAME: new_file_name + } + + # 根据文件名的最后一个部分（输入或输出）确定是添加到input_args还是output + if parts[-3] == KEY_INPUT: + return op_name, CoreConst.INPUT_ARGS, tensor_json + elif parts[-3] == KEY_OUTPUT: + return op_name, KEY_OUTPUT, tensor_json + else: + return None, None, None + + except Exception as e: + logger.error(f"Error reading {file_path}: {e}") + return None, None, None + + +def custom_sort(item, key_to_index): + key = item[0] + return key_to_index.get(key, float('inf')) + + +def generate_dump_info(path): + if not os.path.exists(path): + logger.error("The provided path does not exist.") + return + + dump_data = {"task": "tensor", "level": "L0", "dump_data_dir": path, "data": {}} + + with Pool(processes=10) as pool: + file_paths = [] + for root, _, files in os.walk(path): + for file in files: + if file.endswith(FileCheckConst.NUMPY_SUFFIX): + file_paths.append((os.path.join(root, file),)) + file_paths.sort() + results = pool.starmap(process_file, file_paths) + + # 收集结果 + for op_name, key, tensor_json in results: + if op_name: + if op_name not in dump_data.get(CoreConst.DATA, {}): + dump_data.get(CoreConst.DATA, {})[op_name] = {CoreConst.INPUT_ARGS: [], + CoreConst.INPUT_KWARGS: {}, + KEY_OUTPUT: []} + if key not in dump_data.get(CoreConst.DATA, {}).get(op_name, {}): + dump_data.get(CoreConst.DATA, {}).get(op_name, {})[key] = [] + dump_data.get(CoreConst.DATA, {}).get(op_name, {}).get(key, []).append(tensor_json) + + # 根据cell_list排序 + data_dict = dump_data.get(CoreConst.DATA, {}) + key_to_index = {key: index for index, key in enumerate(cell_list)} + sorted_data_dict = dict(sorted(data_dict.items(), key=lambda item: custom_sort(item, key_to_index))) + dump_data[CoreConst.DATA] = sorted_data_dict + + # 将数据写入dump.json + json_path = os.path.join(os.path.dirname(path), 'dump.json') + save_json(json_path, dump_data, indent=1) + + logger.info(f"Dump data saved to {json_path}") + + +def generate_stack_info(path): + if not os.path.exists(path): + logger.error("The provided path does not exist.") + return + + stack_data = {} + file_paths = [] + # 传入的path为工具生成的./dump_tensor_data，内容为npy文件 + for root, _, files in os.walk(path): + for file in files: + if file.endswith(FileCheckConst.NUMPY_SUFFIX): + file_paths.append(os.path.join(root, file)) + file_paths.sort() + for file_path in file_paths: + # 文件名举例:Cell.network._backbone.loss.CrossEntropyLoss.forward.0.input.0_float32_165.npy + parts = os.path.basename(file_path).split(CoreConst.SEP) + # op_name是Cell.network._backbone.loss.CrossEntropyLoss.forward.0 + op_name = CoreConst.SEP.join(parts[:-3]) + stack_data.update({op_name: []}) + + # 将数据写入stack.json + json_path = os.path.join(os.path.dirname(path), 'stack.json') + save_json(json_path, stack_data, indent=1) + + logger.info(f"Stack data saved to {json_path}") + + +def is_download_finished(directory, interval=3): + """ + 判断指定目录在一段时间后是否有数据被下载完成 + :param directory: 指定目录的路径 + :param interval: 检查的时间间隔（秒），默认为 3 秒 + :return: 如有数据被下载完成返回 True，否则返回 False + """ + # 检查目录是否存在 + if not os.path.exists(directory): + logger.warning(f"The specified directory {directory} does not exist.") + return False + initial_modification_time = os.path.getmtime(directory) + time.sleep(interval) + current_modification_time = os.path.getmtime(directory) + # 比较初始和当前修改时间 + if current_modification_time > initial_modification_time: + return False + else: + return True + + +def process(dump_path): + rank_id = os.environ.get('RANK_ID') + rank_dir = DEFAULT_RANK_DIR + if rank_id is not None: + rank_dir = CoreConst.RANK + str(rank_id) + + step_dir_list = os.listdir(dump_path) + for step_dir in step_dir_list: + step_path = os.path.join(dump_path, step_dir) + rank_path = os.path.join(step_path, rank_dir) + npy_path = os.path.join(rank_path, CoreConst.DUMP_TENSOR_DATA) + while True: + is_finished = is_download_finished(npy_path) + if not is_finished: + logger.info("There is data being downloaded in the specified directory, continue checking...") + else: + logger.info("There is no data being downloaded in the specified directory, Stop checking.") + break + logger.info("==========Start processing data that has already been stored on the disk!==========") + rename_filename(npy_path) + generate_construct(npy_path) + generate_dump_info(npy_path) + generate_stack_info(npy_path) + if rank_id is None: + new_rank_path = os.path.join(step_path, CoreConst.RANK) + try: + os.rename(rank_path, new_rank_path) + logger.info("Directory was successfully renamed to: {new_rank_path}") + except Exception as e: + logger.error(f"Error renamed to {new_rank_path}: {e}") + logger.info("==========JSON file generation completed!==========") + + +def get_yaml_keys(yaml_data): + keys = [] + for key, _ in yaml_data.items(): + keys.append(key) + return keys + + +def get_tensordump_mode(input_str): + left_index = input_str.find('(') + right_index = input_str.find(')') + + # 提取括号内的字符串 + if left_index != -1 and right_index != -1: + inner_str = input_str[left_index + 1:right_index] + # 分割字符串得到元素列表 + elements = inner_str.split(',') + if len(elements) >= 2: + # 去除元素前后的空格 + first_element = elements[0].strip() + second_element = elements[1].strip() + return first_element, second_element + return None, None + + +def str_to_list(input_str): + # 去除首尾的方括号 + input_str = input_str.strip('[]') + # 按逗号分割并去除元素两端的空格 + return [item.strip() for item in input_str.split(',')] + + +def set_tensordump_mode(cell, input_str): + first_str, second_str = get_tensordump_mode(input_str) + inputs_mode = [] + outputs_mode = [] + if first_str and second_str: + inputs_mode = str_to_list(first_str) + outputs_mode = str_to_list(second_str) + if inputs_mode and outputs_mode: + cell.input_dump_mode = inputs_mode + cell.output_dump_mode = outputs_mode + + +def start(net=None, dump_path="./", data_mode=CoreConst.ALL): + if not dump_gradient_op_existed or net is None: + return + + if isinstance(net, nn.Cell): + net = (('', net),) + + td_config_path = "" + try: + import mindformers + mindformers_file = mindformers.__file__ + mindformers_dir = os.path.dirname(mindformers_file) + td_config_path = os.path.join(mindformers_dir, "configuration", "layer_mapping.yaml") + if not os.path.exists(td_config_path): + td_config_path = "" + logger.warning("The configuration file in mindformers was not loaded, the default mode will be used.") + except ImportError: + logger.warning("The mindFormers failed to load, the default mode will be used.") + + if td_config_path == "": + yaml_data = {} + else: + yaml_data = load_yaml(td_config_path) + first_layer_key = get_yaml_keys(yaml_data) + + black_list = ["grad_reducer", ""] + + for name_and_model in net: + for name, cell in name_and_model[1].cells_and_names(name_prefix=name_and_model[0]): + class_name = cell.__class__.__name__ + # 跳过黑名单cell + if name in black_list: + logger.info(f"Cell {name}.{class_name} is skipped!") + continue + # 跳过框架内部的cell + if class_name.startswith(CoreConst.REPLACEMENT_CHARACTER): + logger.info(f"Cell {name}.{class_name} is skipped!") + continue + else: + # Format: Cell.{cell_name}.{class_name} + cell.cell_prefix = CoreConst.SEP.join([CoreConst.CELL, name, cell.__class__.__name__]) + + # 根据yaml配置文件设置cell的TensorDump模式 + if class_name in first_layer_key: + layer_data = yaml_data.get(class_name) + if layer_data: + for child_name, child_cell in cell.cells_and_names(): + if child_name in layer_data: + set_tensordump_mode(child_cell, layer_data[child_name]) + top_layer_data = yaml_data.get(KEY_TOPLAYER) + if top_layer_data and name in top_layer_data: + set_tensordump_mode(cell, top_layer_data[name]) + + # 替换construct函数 + cell.construct = cell_construct_wrapper(cell.construct, cell) + logger.info(f"Cell {name}: construct function is wrapped!") + cell.dump_path = dump_path + cell.data_mode = data_mode + + logger.info("==========The cell_dump_process_start phase is Finished!==========") + atexit.register(process, dump_path=dump_path) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/cell_dump_with_insert_gradient.py b/debug/accuracy_tools/msprobe/mindspore/dump/cell_dump_with_insert_gradient.py new file mode 100644 index 0000000000000000000000000000000000000000..46ca9ba63b3d6815505b304cbb7ce86e5d422232 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/cell_dump_with_insert_gradient.py @@ -0,0 +1,617 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +from multiprocessing import Pool +import os +import re +import time + +import numpy as np +import mindspore as ms +from mindspore import nn, ops + +from msprobe.core.common.const import Const as CoreConst +from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.file_utils import load_npy, save_json, remove_path, load_yaml +from msprobe.mindspore.common.log import logger + + +CONSTRUCT_FILE_NAME = "construct.json" +DEFAULT_RANK_DIR = "rank0" +KEY_LAYERS = "layers" +construct = {} +cell_list = [] +KEY_SIDE_EFFECT = "side_effect_io" +KEY_TOPLAYER = "TopLayer" +KEY_FORWARD = CoreConst.FORWARD +KEY_BACKWARD = CoreConst.BACKWARD +KEY_INPUT = CoreConst.INPUT +KEY_OUTPUT = CoreConst.OUTPUT +td = ops.TensorDump() +if (ms.__version__ >= "2.5.0"): + td_in = ops.TensorDump("in") +else: + td_in = ops.TensorDump() +td.add_prim_attr(KEY_SIDE_EFFECT, False) +td_in.add_prim_attr(KEY_SIDE_EFFECT, False) +np_ms_dtype_dict = { + "bool": ms.bool_, + "int8": ms.int8, + "byte": ms.byte, + "int16": ms.int16, + "short": ms.short, + "int32": ms.int32, + "intc": ms.intc, + "int64": ms.int64, + "intp": ms.intp, + "uint8": ms.uint8, + "ubyte": ms.ubyte, + "uint16": ms.uint16, + "ushort": ms.ushort, + "uint32": ms.uint32, + "uintc": ms.uintc, + "uint64": ms.uint64, + "uintp": ms.uintp, + "float16": ms.float16, + "half": ms.half, + "float32": ms.float32, + "single": ms.single, + "float64": ms.float64, + "double": ms.double, + "bfloat16": ms.bfloat16, + "complex64": ms.complex64, + "complex128": ms.complex128 +} + + +def gen_file_path(dump_path, cell_prefix, suffix, io_type, index): + data_path = os.path.join(dump_path, '{step}', '{rank}', CoreConst.DUMP_TENSOR_DATA) + file_name = cell_prefix + CoreConst.SEP + suffix + CoreConst.SEP + io_type + CoreConst.SEP + str(index) + return os.path.join(data_path, file_name) + + +def partial_func(func, dump_path, cell_prefix, index, io_type): + def newfunc(*args, **kwargs): + return func(dump_path, cell_prefix, index, io_type, *args, **kwargs) + return newfunc + + +def clip_gradient(dump_path, cell_prefix, index, io_type, dx): + if io_type == KEY_OUTPUT: + temp = td(gen_file_path(dump_path, cell_prefix, KEY_BACKWARD, io_type, index), dx) + dx = ops.depend(dx, temp) + elif io_type == KEY_INPUT: + temp = td_in(gen_file_path(dump_path, cell_prefix, KEY_BACKWARD, io_type, index), dx) + dx = ops.depend(dx, temp) + return dx + + +def need_tensordump_in(cell_obj, attr): + return hasattr(cell_obj, attr) and getattr(cell_obj, attr) == "in" + + +def cell_construct_wrapper(func, self): + def new_construct(self, *args, **kwargs): + new_args = [] + out_list = [] + + index = 0 + item = None + backward_or_all = self.data_mode in ["backward", "all"] + forward_or_all = self.data_mode in ["forward", "all"] + # The inputs of the cell. + for index, item in enumerate(args): + if backward_or_all and ops.is_tensor(item): + item = self.output_clips[index](item) + if forward_or_all and ops.is_tensor(item): + if need_tensordump_in(self, 'input_dump_mode'): + temp = td_in( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_INPUT, index), + item + ) + else: + temp = td( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_INPUT, index), + item + ) + item = ops.depend(item, temp) + new_args.append(item) + + out = func(*new_args, **kwargs) + + # The outputs of the cell. + if isinstance(out, tuple): + for index, item in enumerate(out): + if backward_or_all and ops.is_tensor(item): + item = self.input_clips[index](item) + if forward_or_all and ops.is_tensor(item): + if need_tensordump_in(self, 'output_dump_mode'): + temp = td_in( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_OUTPUT, index), + item + ) + else: + temp = td( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_OUTPUT, index), + item + ) + item = ops.depend(item, temp) + out_list.append(item) + elif forward_or_all and not ops.is_tensor(item): + out_list.append(item) + out_list = tuple(out_list) + return out_list + else: + if backward_or_all: + out = self.input_clips[0](out) + if forward_or_all and ops.is_tensor(out): + if need_tensordump_in(self, 'output_dump_mode'): + temp = td_in( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_OUTPUT, 0), + out + ) + else: + temp = td( + gen_file_path(self.dump_path, self.cell_prefix, KEY_FORWARD, KEY_OUTPUT, 0), + out + ) + out = ops.depend(out, temp) + return out + + return new_construct.__get__(self, type(self)) + + +# 获取目录下所有文件名并根据TensorDump落盘自增id从小到大排序 +def sort_filenames(path): + filenames = os.listdir(path) + id_pattern = re.compile(rf'{CoreConst.REPLACEMENT_CHARACTER}(\d+){CoreConst.NUMPY_SUFFIX}$') + filenames.sort(key=lambda x: int(id_pattern.findall(x)[0])) + return filenames + + +# 删除重复dump的文件：自定义文件名相同，并且数据相同 +def del_same_file(path, filenames): + result_list = [] + seen_prefixes = {} + for current_filename in filenames: + parts = current_filename.rsplit(CoreConst.REPLACEMENT_CHARACTER, 1) + prefix = parts[0] + if prefix not in seen_prefixes: + result_list.append(current_filename) + seen_prefixes[prefix] = current_filename + else: + current_file_path = os.path.join(path, current_filename) + current_file = load_npy(current_file_path) + prev_filename = seen_prefixes[prefix] + prev_file_path = os.path.join(path, prev_filename) + prev_file = load_npy(prev_file_path) + if np.array_equal(current_file, prev_file): + remove_path(current_file_path) + logger.warning(f"{current_file_path} is deleted!") + else: + result_list.append(current_filename) + return result_list + + +def rename_filename(path): + filenames = sort_filenames(path) + filenames = del_same_file(path, filenames) + + filename_dict = {} + for filename in filenames: + name_field = filename.rsplit(CoreConst.REPLACEMENT_CHARACTER, 1)[0] + + if name_field in filename_dict: + filename_dict[name_field] += 1 + else: + filename_dict[name_field] = 0 + + cell_index = filename_dict[name_field] + + # 修改文件名，增加重复调用Cell的序号 + if CoreConst.FORWARD_PATTERN in filename: + # Format: Cell.{cell_name}.{class_name}.{forward/backward}.{number}.{input/output}.{index}_{dtype}_{id}.npy + new_file_name = filename.replace(CoreConst.FORWARD_PATTERN, + CoreConst.FORWARD_PATTERN + str(cell_index) + CoreConst.SEP) + if CoreConst.BACKWARD_PATTERN in filename: + new_file_name = filename.replace(CoreConst.BACKWARD_PATTERN, + CoreConst.BACKWARD_PATTERN + str(cell_index) + CoreConst.SEP) + os.rename(os.path.join(path, filename), os.path.join(path, new_file_name)) + logger.info("==========The rename_filename phase is Finished!==========") + + +# Extract the field between the first "." and the third to last ".", i.e. {cell_name} +def get_cell_name(string): + parts = string.split(CoreConst.SEP) + if len(parts) < 4: + return None + start_index = 1 + end_index = len(parts) - 3 + return CoreConst.SEP.join(parts[start_index:end_index]) + + +# Extract the field between the last "." and the second to last ".", i.e. {data_made} +def get_data_mode(string): + last_dot_index = string.rfind(CoreConst.SEP) + second_last_dot_index = string.rfind(CoreConst.SEP, 0, last_dot_index) + data_mode = string[second_last_dot_index + 1:last_dot_index] + return data_mode + + +# 判断二者之间是否存在父子关系 +def check_relation(cell_name, parent_cell_name): + layers_pattern = rf"{CoreConst.SEP}{KEY_LAYERS}{CoreConst.SEP}\d+$" + last_dot_index = cell_name.rfind(CoreConst.SEP) + if last_dot_index == -1: + return False + # 如果cell_name最后一个'.'之前的字段等于parent_cell_name，则判定存在父子关系 + sub_cell_name = cell_name[:last_dot_index] + if sub_cell_name == parent_cell_name: + return True + elif re.search(layers_pattern, cell_name): + # 如果cell_name以".layer.{layer_id}"结尾，且去掉该字段后等于parent_cell_name，则判定存在父子关系 + sub_cell_name = re.sub(layers_pattern, '', cell_name) + if sub_cell_name == parent_cell_name: + return True + return False + + +def get_construct(cell_list_input): + for cell in cell_list_input: + cell_name = get_cell_name(cell) + cell_data_mode = get_data_mode(cell) + found_flag = False + for parent_cell in cell_list_input: + parent_cell_name = get_cell_name(parent_cell) + parent_data_mode = get_data_mode(parent_cell) + has_relation = check_relation(cell_name, parent_cell_name) + if has_relation and parent_data_mode == cell_data_mode: + construct.update({cell: parent_cell}) + found_flag = True + break + if not found_flag: + construct.update({cell: None}) + + +def generate_construct(path): + global construct + filenames = sort_filenames(path) + + # 提取文件名中Cell.{cell_name}.{class_name}.{data_mode}.{重复调用此cell的序号}字段，并存入cell_list + for filename in filenames: + point_position = 3 + mid_field = filename.rsplit(CoreConst.SEP, point_position)[0] + if KEY_INPUT in filename: + if mid_field in cell_list: + cell_list.remove(mid_field) + cell_list.append(mid_field) + else: + if mid_field not in cell_list: + index = filenames.index(filename) + output_field = mid_field + KEY_OUTPUT + find_flag = False + for filename_other in cell_list[index + 1:]: + if output_field in filename_other: + find_flag = True + if find_flag is False: + cell_list.append(mid_field) + + get_construct(cell_list) + + # 生成JSON文件 + rank_dir = os.path.dirname(path) + json_path = os.path.join(rank_dir, CONSTRUCT_FILE_NAME) + save_json(json_path, construct, indent=1) + + # 清空'construct'继续处理下一个路径下的数据 + construct = {} + logger.info(f"Construct data saved to {json_path}") + + +def process_file(file_path): + try: + # 读取.npy文件内容 + npy_content = load_npy(file_path) + logger.debug(f"Loaded {file_path}: shape is {npy_content.shape}, dtype is {npy_content.dtype}") + + # 文件名举例:Cell.network._backbone.loss.CrossEntropyLoss.forward.0.input.0_float32_165.npy + parts = os.path.basename(file_path).split(CoreConst.SEP) + data_dtype = "" + # 获取0_float32_165或者0_in_float32_165中的float32 + data_dtype_list = parts[-2].split('_') + if len(data_dtype_list) > 1: + data_dtype = data_dtype_list[-2] + # op_name是Cell.network._backbone.loss.CrossEntropyLoss.forward.0 + op_name = CoreConst.SEP.join(parts[:-3]) + ms_dtype = np_ms_dtype_dict.get(data_dtype) + if ms_dtype is None: + logger.warning(f"Get dtype None from file {file_path}") + + # 修改落盘文件名字，去掉TensorDump自带的数据类型和自增id字段 + data_file_name = os.path.basename(file_path) + data_file_dir = os.path.dirname(file_path) + parts = data_file_name.split(CoreConst.SEP) + if len(parts) >= 2: + param_index = parts[-2].split(CoreConst.REPLACEMENT_CHARACTER)[0] + pre_parts = CoreConst.SEP.join(parts[:-2]) + new_file_name = pre_parts + CoreConst.SEP + param_index + CoreConst.NUMPY_SUFFIX + os.rename(os.path.join(data_file_dir, data_file_name), os.path.join(data_file_dir, new_file_name)) + logger.debug(f"{data_file_name} is renamed to {new_file_name}") + else: + logger.warning(f"Failed to rename {data_file_name}.") + new_file_name = data_file_name + + tensor_json = { + CoreConst.TYPE: 'mindspore.Tensor', + CoreConst.DTYPE: str(ms_dtype), + CoreConst.SHAPE: list(npy_content.shape), + CoreConst.MAX: npy_content.max().item(), + CoreConst.MIN: npy_content.min().item(), + CoreConst.MEAN: npy_content.mean().item(), + CoreConst.NORM: np.linalg.norm(npy_content).item(), + CoreConst.DATA_NAME: new_file_name + } + + # 根据文件名的最后一个部分（输入或输出）确定是添加到input_args还是output + if parts[-3] == KEY_INPUT: + return op_name, CoreConst.INPUT_ARGS, tensor_json + elif parts[-3] == KEY_OUTPUT: + return op_name, KEY_OUTPUT, tensor_json + else: + return None, None, None + + except Exception as e: + logger.error(f"Error reading {file_path}: {e}") + return None, None, None + + +def custom_sort(item, key_to_index): + key = item[0] + return key_to_index.get(key, float('inf')) + + +def generate_dump_info(path): + if not os.path.exists(path): + logger.error("The provided path does not exist.") + return + + dump_data = {"task": "tensor", "level": "L0", "dump_data_dir": path, "data": {}} + + with Pool(processes=10) as pool: + file_paths = [] + for root, _, files in os.walk(path): + for file in files: + if file.endswith(FileCheckConst.NUMPY_SUFFIX): + file_paths.append((os.path.join(root, file),)) + file_paths.sort() + results = pool.starmap(process_file, file_paths) + + # 收集结果 + for op_name, key, tensor_json in results: + if op_name: + if op_name not in dump_data.get(CoreConst.DATA, {}): + dump_data.get(CoreConst.DATA, {})[op_name] = {CoreConst.INPUT_ARGS: [], + CoreConst.INPUT_KWARGS: {}, + KEY_OUTPUT: []} + if key not in dump_data.get(CoreConst.DATA, {}).get(op_name, {}): + dump_data.get(CoreConst.DATA, {}).get(op_name, {})[key] = [] + dump_data.get(CoreConst.DATA, {}).get(op_name, {}).get(key, []).append(tensor_json) + + # 根据cell_list排序 + data_dict = dump_data.get(CoreConst.DATA, {}) + key_to_index = {key: index for index, key in enumerate(cell_list)} + sorted_data_dict = dict(sorted(data_dict.items(), key=lambda item: custom_sort(item, key_to_index))) + dump_data[CoreConst.DATA] = sorted_data_dict + + # 将数据写入dump.json + json_path = os.path.join(os.path.dirname(path), 'dump.json') + save_json(json_path, dump_data, indent=1) + + logger.info(f"Dump data saved to {json_path}") + + +def generate_stack_info(path): + if not os.path.exists(path): + logger.error("The provided path does not exist.") + return + + stack_data = {} + file_paths = [] + # 传入的path为工具生成的./dump_tensor_data，内容为npy文件 + for root, _, files in os.walk(path): + for file in files: + if file.endswith(FileCheckConst.NUMPY_SUFFIX): + file_paths.append(os.path.join(root, file)) + file_paths.sort() + for file_path in file_paths: + # 文件名举例:Cell.network._backbone.loss.CrossEntropyLoss.forward.0.input.0_float32_165.npy + parts = os.path.basename(file_path).split(CoreConst.SEP) + # op_name是Cell.network._backbone.loss.CrossEntropyLoss.forward.0 + op_name = CoreConst.SEP.join(parts[:-3]) + stack_data.update({op_name: []}) + + # 将数据写入stack.json + json_path = os.path.join(os.path.dirname(path), 'stack.json') + save_json(json_path, stack_data, indent=1) + + logger.info(f"Stack data saved to {json_path}") + + +def is_download_finished(directory, interval=3): + """ + 判断指定目录在一段时间后是否有数据被下载完成 + :param directory: 指定目录的路径 + :param interval: 检查的时间间隔（秒），默认为 3 秒 + :return: 如有数据被下载完成返回 True，否则返回 False + """ + # 检查目录是否存在 + if not os.path.exists(directory): + logger.warning(f"The specified directory {directory} does not exist.") + return False, False + initial_modification_time = os.path.getmtime(directory) + time.sleep(interval) + current_modification_time = os.path.getmtime(directory) + # 比较初始和当前修改时间 + if current_modification_time > initial_modification_time: + return False, True + else: + return True, False + + +def process(dump_path): + rank_id = os.environ.get('RANK_ID') + rank_dir = DEFAULT_RANK_DIR + if rank_id is not None: + rank_dir = CoreConst.RANK + str(rank_id) + + step_dir_list = os.listdir(dump_path) + for step_dir in step_dir_list: + step_path = os.path.join(dump_path, step_dir) + rank_path = os.path.join(step_path, rank_dir) + npy_path = os.path.join(rank_path, CoreConst.DUMP_TENSOR_DATA) + check_times = 0 + while True: + is_finished, is_downloading = is_download_finished(npy_path) + if not is_finished: + if not is_downloading: + logger.warning(f'{npy_path} does not exist.') + break + check_times += 1 + if check_times < 1000: + logger.info("There is data being downloaded in the specified directory, continue checking...") + else: + logger.warning('Download timeout, stop checking.') + break + else: + logger.info("There is no data being downloaded in the specified directory, stop checking.") + break + logger.info("==========Start processing data that has already been stored on the disk!==========") + rename_filename(npy_path) + generate_construct(npy_path) + generate_dump_info(npy_path) + generate_stack_info(npy_path) + if rank_id is None: + new_rank_path = os.path.join(step_path, CoreConst.RANK) + try: + os.rename(rank_path, new_rank_path) + logger.debug(f"Directory was successfully renamed to: {new_rank_path}") + except Exception as e: + logger.error(f"Error renamed to {new_rank_path}: {e}") + logger.info("==========JSON file generation completed!==========") + + +def get_yaml_keys(yaml_data): + keys = [] + for key, _ in yaml_data.items(): + keys.append(key) + return keys + + +def get_tensordump_mode(input_str): + left_index = input_str.find('(') + right_index = input_str.find(')') + + # 提取括号内的字符串 + if left_index != -1 and right_index != -1: + inner_str = input_str[left_index + 1:right_index] + # 分割字符串得到元素列表 + elements = inner_str.split(',') + if len(elements) >= 2: + # 去除元素前后的空格 + first_element = elements[0].strip() + second_element = elements[1].strip() + return first_element, second_element + return None, None + + +def set_tensordump_mode(cell, input_str): + first_str, second_str = get_tensordump_mode(input_str) + if first_str and second_str: + cell.input_dump_mode = first_str + cell.output_dump_mode = second_str + + +def start(net=None, dump_path="./", data_mode=CoreConst.ALL): + if net is None: + return + + if isinstance(net, nn.Cell): + net = (('', net),) + + td_config_path = "" + try: + import mindformers + mindformers_file = mindformers.__file__ + mindformers_dir = os.path.dirname(mindformers_file) + td_config_path = os.path.join(mindformers_dir, "configuration", "layer_mapping.yaml") + if not os.path.exists(td_config_path): + td_config_path = "" + logger.warning("The configuration file in mindformers was not loaded, the default mode will be used.") + except ImportError: + logger.warning("The mindFormers failed to load, the default mode will be used.") + + if td_config_path == "": + yaml_data = {} + else: + yaml_data = load_yaml(td_config_path) + first_layer_key = get_yaml_keys(yaml_data) + + black_list = ["grad_reducer", ""] + + for name_and_model in net: + for name, cell in name_and_model[1].cells_and_names(name_prefix=name_and_model[0]): + class_name = cell.__class__.__name__ + # 跳过黑名单cell + if name in black_list: + logger.info(f"Cell {name}.{class_name} is skipped!") + continue + # 跳过框架内部的cell + if class_name.startswith(CoreConst.REPLACEMENT_CHARACTER): + logger.info(f"Cell {name}.{class_name} is skipped!") + continue + else: + # Format: Cell.{cell_name}.{class_name} + cell.cell_prefix = CoreConst.SEP.join([CoreConst.CELL, name, cell.__class__.__name__]) + + # 根据yaml配置文件设置cell的TensorDump模式 + if class_name in first_layer_key: + layer_data = yaml_data.get(class_name) + if layer_data: + for child_name, child_cell in cell.cells_and_names(): + if child_name in layer_data: + set_tensordump_mode(child_cell, layer_data[child_name]) + top_layer_data = yaml_data.get(KEY_TOPLAYER) + if top_layer_data and name in top_layer_data: + set_tensordump_mode(cell, top_layer_data[name]) + + # 替换construct函数 + cell.construct = cell_construct_wrapper(cell.construct, cell) + logger.info(f"Cell {name}: construct function is wrapped!") + cell.dump_path = dump_path + cell.data_mode = data_mode + cell.input_clips = [] + cell.output_clips = [] + # It is assumed that each cell has a maximum of 50 outputs and 50 inputs. + for i in range(50): + cell.input_clips.append( + ops.InsertGradientOf(partial_func(clip_gradient, cell.dump_path, cell.cell_prefix, i, KEY_INPUT)) + ) + cell.output_clips.append( + ops.InsertGradientOf(partial_func(clip_gradient, cell.dump_path, cell.cell_prefix, i, KEY_OUTPUT)) + ) + + logger.info("==========The cell_dump_process_start phase is Finished!==========") + atexit.register(process, dump_path=dump_path) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py index 0ca63b4a84aee00127bca37b7da36888e905a5aa..06c3226198da22706a8b6d768c26a4beae5a4a54 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,15 +14,18 @@ # limitations under the License. from msprobe.mindspore.common.const import Const +from msprobe.core.common.log import logger +from msprobe.mindspore.common.utils import is_graph_mode_cell_dump_allowed from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.kernel_graph_dump import KernelGraphDump from msprobe.mindspore.dump.kernel_kbyk_dump import KernelKbykDump +from msprobe.mindspore.dump.graph_mode_cell_dump import GraphModeCellDump class DumpToolFactory: tools = { Const.CELL: { - Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_KBYK_MODE: GraphModeCellDump, Const.GRAPH_GE_MODE: None, Const.PYNATIVE_MODE: None }, @@ -39,14 +42,21 @@ class DumpToolFactory: } @staticmethod - def create(config: DebuggerConfig): - if len(config.data_mode) != 1 or config.data_mode[0] not in Const.GRAPH_DATA_MODE_LIST: - raise Exception("data_mode must be one of all, input, output.") + def create(config: DebuggerConfig, model=None): + if config.level == Const.CELL: + if not is_graph_mode_cell_dump_allowed(config): + raise Exception("Cell dump is not supported in graph mode.") + if len(config.data_mode) != 1 or config.data_mode[0] not in Const.GRAPH_CELL_DUMP_DATA_MODE_LIST: + raise Exception("data_mode must be one of all, forward, backward.") + else: + if len(config.data_mode) != 1 or config.data_mode[0] not in Const.GRAPH_DATA_MODE_LIST: + raise Exception("data_mode must be one of all, input, output.") tool = DumpToolFactory.tools.get(config.level) if not tool: raise Exception("Valid level is needed.") tool = tool.get(config.execution_mode) if not tool: - raise Exception(f"Data dump is not supported in {config.execution_mode} mode " - f"when dump level is {config.level}.") - return tool(config) + logger.error(f"Data dump is not supported in {config.execution_mode} mode " + f"when dump level is {config.level}.") + raise ValueError + return tool(config, model) if tool == GraphModeCellDump else tool(config) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/graph_mode_cell_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/graph_mode_cell_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..4a79c96ccb652777e41498a48e7a13d988b288ed --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/graph_mode_cell_dump.py @@ -0,0 +1,121 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import mindspore as ms +from mindspore import hal, ops, Tensor +from mindspore.ops.primitive import _run_op + +from msprobe.core.common.const import Const as CoreConst +from msprobe.mindspore.common.const import Const +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +import msprobe.mindspore.dump.cell_dump_process as cellDumperWithDumpGradient +import msprobe.mindspore.dump.cell_dump_with_insert_gradient as cellDumperWithInsertGradient +from msprobe.mindspore.runtime import Runtime + +tensordump_flag = True +try: + from mindspore._c_expression import _tensordump_set_step +except ImportError: + tensordump_flag = False + + +class GraphModeCellDump: + def __init__(self, config: DebuggerConfig, model, strict=True): + self.net = model + self.white_list = [] + self.black_list = [] + self.execution_mode = config.execution_mode + self.dump_path = config.dump_path if config.dump_path else "./" + self.rank = config.rank + self.step = config.step + self.scope = config.scope + self.list = config.list + self.data_mode = config.data_mode + self.file_format = config.file_format + self.check_config(strict) + self.set_step() + + @staticmethod + def step(): + hal.synchronize() + temp_tensor = ms.Tensor([1], dtype=ms.float32) + step_flag = "" + _run_op(ops.TensorDump(), "TensorDump", (step_flag, temp_tensor)) + ops.tensordump(step_flag, temp_tensor) + + def check_config(self, strict): + if not self.net: + raise Exception("The model is empty and cell dump is not enabled.") + + if strict: + if self.rank: + raise Exception("In graph mode, cell dump does not currently support specifying rank.") + if self.scope: + raise Exception("In graph mode, cell dump does not currently support specifying scope.") + if self.list: + raise Exception("In graph mode, cell dump does not currently support specifying list.") + if len(self.data_mode) != 1 or self.data_mode[0] not in Const.GRAPH_CELL_DUMP_DATA_MODE_LIST: + raise Exception("In graph mode and cell dump, data_mode must be one of all, forword, backword.") + if self.file_format != []: + logger.warning("In graph mode, cell dump does not currently support specifying file_format." + " The file will be stored in npy format.") + else: + self.rank = [] + self.scope = [] + self.list = [] + self.file_format = [] + if len(self.data_mode) != 1 or self.data_mode[0] not in Const.GRAPH_CELL_DUMP_DATA_MODE_LIST: + self.data_mode = [CoreConst.ALL] + + return True + + def set_step(self): + if tensordump_flag: + _tensordump_set_step(self.step) + else: + raise Exception( + "Importing _tensordump_set_step failed, " + "please use the latest version package of MindSpore." + ) + + def handle(self): + os.environ['MS_JIT_MODULES'] = 'msprobe' + + if Runtime.run_mode == Const.PYNATIVE_GRAPH_MODE: + dump_path = os.path.join(self.dump_path, Const.GRAPH_MODE) + else: + dump_path = self.dump_path + + cell_dumper = cellDumperWithDumpGradient + + if self.execution_mode == Const.PYNATIVE_MODE: + enable_dump_gradient = hasattr(ops, 'DumpGradient') + if hasattr(ops, 'DumpGradient'): + try: + ops.DumpGradient()('grad.npy', Tensor([0], dtype=ms.float32), 'in') + except Exception: + enable_dump_gradient = False + logger.warning('the DumpGradient operator failed to execute.') + if not enable_dump_gradient: + cell_dumper = cellDumperWithInsertGradient + + cell_dumper.start( + net=self.net, + dump_path=dump_path, + data_mode=self.data_mode[0] + ) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/graph_tensor_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/graph_tensor_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..7b3f249e7e7065d52046aa6991a9d8553bb230d6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/graph_tensor_dump.py @@ -0,0 +1,123 @@ +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from collections import OrderedDict +import mindspore as ms + + +def _iterate_items(data): + if isinstance(data, (dict, OrderedDict)): + return data.items() + elif isinstance(data, (list, tuple)): + return enumerate(data) + else: + raise TypeError("Unsupported data type") + + +class _SaveBase: + def __init__(self, save_dir): + super(_SaveBase, self).__init__() + self.path = save_dir + self.save_func = _npy_save + + def get_save_func(self): + return self.save_func + + +@ms.jit_class +class _SaveCell(_SaveBase): + def __call__(self, name, data): + return self.get_save_func()(self.path, name, data) + + +class _SaveGradBase: + def __init__(self, save_dir, name): + super(_SaveGradBase, self).__init__() + self.file = save_dir + name + + +@ms.jit_class +class _SaveGradCell(_SaveGradBase): + def __init__(self, save_dir, name): + super(_SaveGradCell, self).__init__(save_dir, name) + self.ms_save_grad = ms.ops.InsertGradientOf( + _wrapper_save_grad_func(self.file)) + + def __call__(self, x): + if isinstance(x, ms.Tensor): + return self.ms_save_grad(x) + else: + raise TypeError(f"For 'save_grad', the type of argument 'data' must be mindspore.Tensor or torch.tensor, " + f"but got {type(x)}") + + +def _npy_save_ops(file, data): + if isinstance(data, ms.Tensor): + if data.dtype == ms.bfloat16: + data = data.float() + ms.ops.TensorDump()(file, data) + else: + raise TypeError(f"For 'save', the type of argument 'data' must be mindspore.Tensor or torch.tensor, " + f"but got {type(data)}") + + +def _wrapper_save_grad_func(file): + def _save_grad_func(grad): + data = grad + if data.dtype == ms.bfloat16: + data = data.float() + ms.ops.TensorDump()(file, data) + return grad + return _save_grad_func + + +def _npy_save(save_dir, item_name, data): + if isinstance(data, (list, tuple, dict, OrderedDict)): + for key, val in _iterate_items(data): + _npy_save(save_dir, f"{item_name}.{key}", val) + else: + if data is None: + return + _npy_save_ops(f"{save_dir}{item_name}", data) + + +def generate_dump_dir(save_dir, sep=os.sep): + """ + usage: generate dump directory path str in mindspore graph mode + """ + full_suffix = '{step}' + sep + '{rank}' + sep + if save_dir and save_dir[-1] != sep: + result_dir = save_dir + sep + full_suffix + else: + result_dir = save_dir + full_suffix + return result_dir + + +def save(save_dir, name, data): + """ + save tensor. + """ + dump_dir = generate_dump_dir(save_dir) + _SaveCell(dump_dir)(name, data) + + +def save_grad(save_dir, name, data): + """ + save grad. + """ + dump_dir = generate_dump_dir(save_dir) + suffix_name = name + '_grad' + return _SaveGradCell(dump_dir, suffix_name)(data) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_register.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_register.py index 7a5737662d4e6619d90a6744f975d49fe1784825..6b564f0e70b2d3cd184e879cf44e6a090d50bb81 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_register.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_register.py @@ -14,14 +14,17 @@ # limitations under the License. import os +import inspect from mindspore import Tensor, ops, mint +from mindspore.mint import distributed from mindspore.mint.nn import functional from mindspore.communication import comm_func from msprobe.core.common.file_utils import load_yaml from msprobe.core.common.utils import Const from msprobe.core.data_dump.api_registry import ApiRegistry +from msprobe.mindspore.common.log import logger from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.common.utils import is_mindtorch from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell @@ -41,7 +44,8 @@ if not is_mindtorch(): Const.MS_API_TYPE_TENSOR: (Tensor, (Tensor,)), Const.MS_API_TYPE_MINT: (mint, (mint,)), Const.MS_API_TYPE_MINT_FUNC: (functional, (functional,)), - Const.MS_API_TYPE_COM: (comm_func, (comm_func,)) + Const.MS_API_TYPE_COM: (comm_func, (comm_func,)), + Const.MS_API_TYPE_MINT_DIST: (distributed, (distributed,)) } } if stub_tensor_existed: @@ -50,6 +54,7 @@ if not is_mindtorch(): ) _supported_api_list_path = (os.path.join(cur_path, MsConst.SUPPORTED_API_LIST_FILE),) + _backlist = [] else: import torch import torch_npu @@ -64,13 +69,14 @@ else: } _supported_api_list_path = (os.path.join(cur_path, '../../../pytorch/hook_module', MsConst.SUPPORTED_API_LIST_FILE),) + _backlist = [f'{Const.PT_API_TYPE_TENSOR}.__setitem__'] _inner_used_api = { Const.MS_FRAMEWORK + Const.SEP + Const.MS_API_TYPE_OPS: ( ops, "norm", "square", "sqrt", "is_complex", "stack", "is_floating_point" ), Const.MS_FRAMEWORK + Const.SEP + Const.MS_API_TYPE_TENSOR: ( - Tensor, "to", "numel" + Tensor, "to", "numel", 'sum' ), Const.MS_FRAMEWORK + Const.SEP + Const.MS_API_TYPE_MINT: ( mint, "max", "min", "mean", "norm" @@ -84,6 +90,9 @@ class ApiTemplate(HOOKCell): self.api_func = api_func self.prefix_api_name = prefix + Const.SEP + str(api_name.split(Const.SEP)[-1]) + Const.SEP super().__init__(hook_build_func) + distributed_prefix = Const.DIST_API_TYPE_PREFIX if is_mindtorch() else Const.MINT_DIST_API_TYPE_PREFIX + if prefix == distributed_prefix: + self.op_is_distributed = True @staticmethod def async_to_sync(output): @@ -103,9 +112,22 @@ class ApiTemplate(HOOKCell): output = self.api_func(*args, **kwargs) - if self.prefix_api_name.startswith(MsConst.DISTRIBUTED_DATA_PREFIX): - if kwargs.get("async_op") or self.api_name in ["isend", "irecv"]: + if self.prefix_api_name.startswith( + (MsConst.DISTRIBUTED_DATA_PREFIX, Const.MINT_DIST_API_TYPE_PREFIX) + ): + try: + bound = inspect.signature(self.api_func).bind(*args, **kwargs) + bound.apply_defaults() + use_asyn_op_flag = bound.arguments.get("asyn_op", False) + except Exception as e: + use_asyn_op_flag = False + logger.warning(f"fail to get dist api's func signature because {e}, no wait") + + if use_asyn_op_flag or self.api_name in ["isend", "irecv"]: output = self.async_to_sync(output) + if self.api_name == "batch_isend_irecv" and isinstance(output, list): + output = [self.async_to_sync(handle) for handle in output] + return output def forward(self, *args, **kwargs): @@ -134,9 +156,21 @@ def get_api_register(return_new=False): stub_tensor_set = True if return_new: - return ApiRegistry(_api_types, _inner_used_api, _supported_api_list_path, ApiTemplate) + return ApiRegistry( + _api_types, + _inner_used_api, + _supported_api_list_path, + ApiTemplate, + _backlist + ) global api_register if api_register is None: - api_register = ApiRegistry(_api_types, _inner_used_api, _supported_api_list_path, ApiTemplate) + api_register = ApiRegistry( + _api_types, + _inner_used_api, + _supported_api_list_path, + ApiTemplate, + _backlist + ) return api_register diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py index 7007992ca4540a06b1ebc85a068179e88ec589cc..f19e1e9e4c4aaccce9558fd5a2dce8cebc417cb7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py @@ -31,7 +31,7 @@ def get_cell_count(name): def __init__(self, hook_build_func) -> None: super(HOOKCell, self).__init__() self.changed_status = False - self.input_kwargs = {} + self.msprobe_input_kwargs = {} if not HOOKCell.g_stop_hook: HOOKCell.g_stop_hook = True self.changed_status = True @@ -49,7 +49,7 @@ def __init__(self, hook_build_func) -> None: # 重载call，加全局标志。 def __call__(self, *args, **kwargs): try: - self.input_kwargs = kwargs + self.msprobe_input_kwargs = kwargs out = super(HOOKCell, self).__call__(*args, **kwargs) except Exception as e: raise e diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/primitive_hooks.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/primitive_hooks.py index 656e48c678956563a6f2d1d5f5ab8a4d03f074e7..4b187e13148b06d0429983522cbca443c29ec5d7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/primitive_hooks.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/primitive_hooks.py @@ -58,7 +58,7 @@ class PrimitiveHookService: def backward_hook(grad): captured_grads.extend(grad) backward_primitive_name = f"{updated_primitive_name}{Const.SEP}{Const.BACKWARD}" - + self.service_instance.inner_switch = True try: if hook_type == Const.INPUT: self.service_instance.data_collector.update_api_or_module_name(backward_primitive_name) @@ -77,6 +77,7 @@ class PrimitiveHookService: logger.error(f"This is a primitive op {hook_type}_backward dump error: {exception}, " f"updated_primitive_name: {updated_primitive_name}") raise DumpException(DumpException.BACKWARD_DATA_COLLECTION_ERROR) from exception + self.service_instance.inner_switch = False return backward_hook @@ -137,6 +138,7 @@ class PrimitiveHookService: def pre_forward_hook(primitive_name, primitive_instance, args, kwargs): module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=None) + self.service_instance.inner_switch = True try: self.service_instance.data_collector.forward_input_data_collect( primitive_name, @@ -148,9 +150,11 @@ class PrimitiveHookService: logger.error(f"This is a primitive op dump error during forward input data collection: {exception}, " f"primitive_name: {primitive_name}") raise DumpException(DumpException.FORWARD_DATA_COLLECTION_ERROR) from exception + self.service_instance.inner_switch = False def post_forward_hook(primitive_name, primitive_instance, args, kwargs, output): module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=output) + self.service_instance.inner_switch = True try: self.service_instance.data_collector.forward_output_data_collect( primitive_name, @@ -162,6 +166,7 @@ class PrimitiveHookService: logger.error(f"This is a primitive op dump error during forward output data collection: {exception}, " f"primitive_name: {primitive_name}") raise DumpException(DumpException.FORWARD_DATA_COLLECTION_ERROR) from exception + self.service_instance.inner_switch = False def wrapped_primitive_call(instance_self, *args, **kwargs): """ @@ -179,7 +184,7 @@ class PrimitiveHookService: current_count = self.primitive_counters.get(primitive_name, 0) updated_primitive_name = f"{Const.PRIMITIVE_PREFIX}{Const.SEP}{primitive_name}{Const.SEP}{current_count}" - if not self.service_instance.primitive_switch: + if not self.service_instance.primitive_switch or self.service_instance.inner_switch: return origin_func(*args, **kwargs) captured_grads_input, captured_grads_output = [], [] diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml index 364062b46478b63369269c2470ea526eec59a3d3..eae8f85a87fb2b0986cefb2e6faae7399a86f367 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml @@ -1025,3 +1025,21 @@ communication.comm_func: - recv - isend - irecv + +mint.distributed: + - send + - recv + - broadcast + - all_reduce + - reduce + - all_gather + - gather + - isend + - irecv + - scatter + - reduce_scatter + - all_to_all_single + - all_to_all + - all_gather_into_tensor + - reduce_scatter_tensor + - batch_isend_irecv diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py index 634b15767528da447adadbe324aa4163adc14838..528b522abc3958348804ba77c69832065ffeeb95 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from collections import defaultdict +import os +import types import mindspore +from mindspore import nn from mindspore._c_expression import PyNativeExecutor_ try: from mindspore.common.api import _MindsporeFunctionExecutor @@ -26,7 +28,9 @@ except ImportError: from msprobe.core.common.log import logger from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs +from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.dump.hook_cell.api_register import get_api_register +from msprobe.mindspore.runtime import Runtime _api_register = get_api_register() @@ -34,24 +38,20 @@ _api_register = get_api_register() def dump_jit(name, in_feat, out_feat, is_forward): pid = os.getpid() - ori_args = str(name) - index = ori_args.find("<") - if index != 0 and index != -1: - result = ori_args[0:index] - elif name is not None and "<" not in str(name): - result = str(name) - else: - result = "JitFunction" + name = name if name else "JitFunction" if JitDump.need_dump(): if is_forward: - JitDump.jit_count[result] += 1 - name_template = (Const.JIT + Const.SEP + result + Const.SEP + - str(JitDump.jit_count[result]) + Const.SEP + Const.FORWARD) + if name in JitDump.jit_count: + JitDump.jit_count[name] += 1 + else: + JitDump.jit_count[name] = 0 + name_template = (Const.JIT + Const.SEP + name + Const.SEP + + str(JitDump.jit_count[name]) + Const.SEP + Const.FORWARD) JitDump.data_collector.update_api_or_module_name(name_template) module_input_output = ModuleForwardInputsOutputs(args=in_feat, kwargs={}, output=out_feat) JitDump.data_collector.forward_data_collect(name_template, None, pid, module_input_output) else: - name_template = Const.JIT + Const.SEP + result + Const.SEP + str(JitDump.jit_count[result]) + Const.SEP + \ + name_template = Const.JIT + Const.SEP + name + Const.SEP + str(JitDump.jit_count[name]) + Const.SEP + \ Const.BACKWARD JitDump.data_collector.update_api_or_module_name(name_template) module_input_output = ModuleBackwardInputsOutputs(grad_input=in_feat, grad_output=out_feat) @@ -61,7 +61,7 @@ def dump_jit(name, in_feat, out_feat, is_forward): class JitDump(_MindsporeFunctionExecutor): dump_config = None jit_enable = False - jit_dump_switch = True + jit_dump_switch = False jit_count = defaultdict(int) def __init__(self, *args, **kwargs): @@ -72,19 +72,17 @@ class JitDump(_MindsporeFunctionExecutor): self._executor = PyNativeExecutor_.get_instance() def __call__(self, *args, **kwargs): - if JitDump.jit_dump_switch: - _api_register.restore_all_api() + _api_register.restore_all_api() out = super().__call__(*args, **kwargs) - if JitDump.jit_dump_switch and len(args) > 0: - if self.name and self.name != "construct": + if JitDump.jit_dump_switch and len(args) > 0 and self.name: + if self.name != "construct": dump_jit(self.name, args, out, True) - else: - dump_jit(args[0], args, out, True) + elif Runtime.run_mode != MsConst.PYNATIVE_GRAPH_MODE and isinstance(args[0], nn.Cell): + dump_jit(args[0].__class__.__name__, args, out, True) JitDump.jit_enable = True elif len(args) == 0: logger.warning(f"The jit function {self.name} has no input arguments, nothing will be dumped.") - if JitDump.jit_dump_switch: - _api_register.register_all_api() + _api_register.register_all_api() return out @classmethod @@ -111,6 +109,9 @@ class JitDump(_MindsporeFunctionExecutor): else: output = self._executor.grad(grad, obj, weights, grad_position, *args, *(kwargs.values())) if JitDump.jit_dump_switch and JitDump.jit_enable: - dump_jit(obj, args, None, False) + if isinstance(obj, types.FunctionType): + dump_jit(obj.__name__, args, None, False) + elif Runtime.run_mode != MsConst.PYNATIVE_GRAPH_MODE and isinstance(obj, nn.Cell): + dump_jit(obj.__class__.__name__, args, None, False) _api_register.register_all_api() return output diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py index 2c46b0c73e7789ea41afb991bb985e089b2349cd..e29b05330b7c98a69cc31cadc6583ea11b24c707 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py @@ -39,9 +39,12 @@ class KernelKbykDump: common_set["input_output"] = 0 common_set["kernels"] = [] common_set["support_device"] = [0, 1, 2, 3, 4, 5, 6, 7] - e2e_set = dict() - e2e_set["enable"] = True - e2e_set["trans_flag"] = True + e2e_set = { + "enable": not config.async_dump, + "trans_flag": True, + "stat_calc_mode": config.stat_cal_mode, + "device_stat_precision_mode": config.device_stat_precision_mode, + } if config.list: common_set["dump_mode"] = 1 diff --git a/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.cc b/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.cc deleted file mode 100644 index 9ef4eec3ad4855f91e5b93322f3009afe2b42a41..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.cc +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Copyright 2024 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "hook_dynamic_loader.h" -#include -#include -#include -#include -#include "utils/log_adapter.h" - -namespace py = pybind11; - -HookDynamicLoader &HookDynamicLoader::GetInstance() { - static HookDynamicLoader instance; - return instance; -} - -bool HookDynamicLoader::loadFunction(void *handle, const std::string &functionName) { - void *func = dlsym(handle, functionName.c_str()); - if (!func) { - MS_LOG(WARNING) << "Could not load function: " << functionName << ", error: " << dlerror(); - return false; - } - funcMap_[functionName] = func; - return true; -} - -bool HookDynamicLoader::LoadLibrary() { - std::string msprobePath = ""; - // 获取gil锁 - py::gil_scoped_acquire acquire; - try { - py::module msprobeMod = py::module::import("msprobe.lib._msprobe_c"); - if (!py::hasattr(msprobeMod, "__file__")) { - MS_LOG(WARNING) << "Adump mod not found"; - return false; - } - msprobePath = msprobeMod.attr("__file__").cast(); - } catch (const std::exception& e) { - MS_LOG(WARNING) << "Adump mod path unable to get: " << e.what(); - return false; - } - std::lock_guard lock(mutex_); - if (handle_) { - MS_LOG(WARNING) << "Hook library already loaded!"; - return false; - } - if (msprobePath == "") { - MS_LOG(WARNING) << "Adump path not loaded"; - return false; - } - handle_ = dlopen(msprobePath.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle_) { - MS_LOG(WARNING) << "Failed to load Hook library: " << dlerror(); - return false; - } - - for (const auto &functionName : functionList_) { - if (!loadFunction(handle_, functionName)) { - MS_LOG(WARNING) << "Failed to load adump function"; - dlclose(handle_); - handle_ = nullptr; - return false; - } - } - - MS_LOG(INFO) << "Hook library loaded successfully."; - return true; -} - -bool HookDynamicLoader::UnloadLibrary() { - std::lock_guard lock(mutex_); - if (!handle_) { - MS_LOG(WARNING) << "Hook library hasn't been loaded."; - return false; - } - - dlclose(handle_); - handle_ = nullptr; - funcMap_.clear(); - MS_LOG(INFO) << "Library unloaded successfully."; - return true; -} - -void *HookDynamicLoader::GetHooker(const std::string &funcName) { - std::lock_guard lock(mutex_); - auto iter = funcMap_.find(funcName); - if (iter == funcMap_.end()) { - MS_LOG(WARNING) << "Function not found: " << funcName; - return nullptr; - } - return iter->second; -} diff --git a/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.cpp b/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6cd3d0c75b4e9e2ca8000db0866bfeaa5958a66f --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.cpp @@ -0,0 +1,110 @@ +/** + * Copyright 2024 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hook_dynamic_loader.h" +#include +#include +#include +#include +#include "utils/log_adapter.h" + +namespace py = pybind11; + +HookDynamicLoader &HookDynamicLoader::GetInstance() +{ + static HookDynamicLoader instance; + return instance; +} + +bool HookDynamicLoader::LoadFunction(void *handle, const std::string &functionName) { + void *func = dlsym(handle, functionName.c_str()); + if (!func) { + MS_LOG(WARNING) << "Could not load function: " << functionName << ", error: " << dlerror(); + return false; + } + funcMap_[functionName] = func; + return true; +} + +bool HookDynamicLoader::LoadLibrary() +{ + std::string msprobePath = ""; + // 获取gil锁 + py::gil_scoped_acquire acquire; + try { + py::module msprobeMod = py::module::import("msprobe.lib._msprobe_c"); + if (!py::hasattr(msprobeMod, "__file__")) { + MS_LOG(WARNING) << "Adump mod not found"; + return false; + } + msprobePath = msprobeMod.attr("__file__").cast(); + } catch (const std::exception& e) { + MS_LOG(WARNING) << "Adump mod path unable to get: " << e.what(); + return false; + } + std::lock_guard lock(mutex_); + if (handle_) { + MS_LOG(WARNING) << "Hook library already loaded!"; + return false; + } + if (msprobePath == "") { + MS_LOG(WARNING) << "Adump path not loaded"; + return false; + } + handle_ = dlopen(msprobePath.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle_) { + MS_LOG(WARNING) << "Failed to load Hook library: " << dlerror(); + return false; + } + + for (const auto &functionName : functionList_) { + if (!LoadFunction(handle_, functionName)) { + MS_LOG(WARNING) << "Failed to load adump function"; + dlclose(handle_); + handle_ = nullptr; + return false; + } + } + + MS_LOG(INFO) << "Hook library loaded successfully."; + return true; +} + +bool HookDynamicLoader::UnloadLibrary() +{ + std::lock_guard lock(mutex_); + if (!handle_) { + MS_LOG(WARNING) << "Hook library hasn't been loaded."; + return false; + } + + dlclose(handle_); + handle_ = nullptr; + funcMap_.clear(); + MS_LOG(INFO) << "Library unloaded successfully."; + return true; +} + +void *HookDynamicLoader::GetHooker(const std::string &funcName) +{ + std::lock_guard lock(mutex_); + auto iter = funcMap_.find(funcName); + if (iter == funcMap_.end()) { + MS_LOG(WARNING) << "Function not found: " << funcName; + return nullptr; + } + return iter->second; +} diff --git a/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.h b/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.h index 3e604558aee825e69dcffc00b817f7980e17e3e1..f1bcd84e70bc4a1e3bf4e164eb3da6374a60b3b6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.h +++ b/debug/accuracy_tools/msprobe/mindspore/dym_loader/hook_dynamic_loader.h @@ -27,26 +27,26 @@ constexpr auto kHookBegin = "MS_DbgOnStepBegin"; constexpr auto kHookEnd = "MS_DbgOnStepEnd"; class HookDynamicLoader { - public: - static HookDynamicLoader &GetInstance(); +public: + static HookDynamicLoader &GetInstance(); - HookDynamicLoader(const HookDynamicLoader &) = delete; - HookDynamicLoader &operator=(const HookDynamicLoader &) = delete; + HookDynamicLoader(const HookDynamicLoader &) = delete; + HookDynamicLoader &operator=(const HookDynamicLoader &) = delete; - bool LoadLibrary(); - bool UnloadLibrary(); - void *GetHooker(const std::string &funcName); + bool LoadLibrary(); + bool UnloadLibrary(); + void *GetHooker(const std::string &funcName); - private: - // Helper functions - bool loadFunction(void *handle, const std::string &functionName); +private: + // Helper functions + bool LoadFunction(void *handle, const std::string &functionName); - HookDynamicLoader() = default; + HookDynamicLoader() = default; - void *handle_ = nullptr; - std::vector functionList_ = {kHookBegin, kHookEnd}; - std::map funcMap_; - std::mutex mutex_; + void *handle_ = nullptr; + std::vector functionList_ = {kHookBegin, kHookEnd}; + std::map funcMap_; + std::mutex mutex_; }; #endif // HOOK_DYNAMIC_LOADER_H diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index da4821b3ac45a689fab5ba5c63515f88bd6e17c3..8a2f5d3b6b35843801baad30c17acb4debb50760 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -75,7 +75,7 @@ class ApiPyNativeSelfCheck: ret = None if not need_wrapper_func(): - del cell.input_kwargs + del cell.msprobe_input_kwargs return ret api_name_with_id = api_name_with_id[:-1] @@ -84,9 +84,9 @@ class ApiPyNativeSelfCheck: api_name_with_id[api_name_with_id.find(Const.SEP) + 1:api_name_with_id.rfind(Const.SEP)]) if api_name in self.api_list: ret = check_self(api_name_with_id, output_data, self.ori_func.get(api_name), - *input_data, **cell.input_kwargs) + *input_data, **cell.msprobe_input_kwargs) - del cell.input_kwargs + del cell.msprobe_input_kwargs return ret def backward_hook(cell, grad_input, grad_output): diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py index 3fd1430bff792d5043429caac8fe477e457b8bee..39ca164f2043c5d8f6d2e05987edfffe5bca2bee 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,7 @@ # limitations under the License. from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.mindspore.common.log import logger from msprobe.mindspore.free_benchmark.common.config import Config from msprobe.mindspore.free_benchmark.perturbation.add_noise import AddNoisePerturbation from msprobe.mindspore.free_benchmark.perturbation.bit_noise import BitNoisePerturbation @@ -41,4 +42,5 @@ class PerturbationFactory: if perturbation: return perturbation(api_name_with_id) else: - raise Exception(f'{Config.pert_type} is a invalid perturbation type') + logger.error(f'{Config.pert_type} is a invalid perturbation type') + raise ValueError diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py index 35b5eb2ab65511fa4320dc97702a60a9c8d07f62..b21b15d1758a90e62861c7edf2976d38ab43c5f0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,7 @@ # limitations under the License. from msprobe.mindspore.common.const import Const +from msprobe.core.common.log import logger from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelfCheck @@ -41,8 +42,10 @@ class SelfCheckToolFactory: def create(config: DebuggerConfig): tool = SelfCheckToolFactory.tools.get(config.level) if not tool: - raise Exception(f"{config.level} is not supported.") + logger.error(f"{config.level} is not supported.") + raise ValueError tool = tool.get(config.execution_mode) if not tool: - raise Exception(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.") + logger.error(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.") + raise ValueError return tool(config) diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py index 01e46e019a4d1634a4592970386d855637c34e8f..ca032e61e5b5cc0d98732ac0bca2d14f377ebfb1 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -16,6 +16,7 @@ import os import threading from typing import Dict, Union, Tuple +import time from msprobe.core.common.utils import is_int from msprobe.core.common.file_utils import create_directory, check_path_before_create @@ -69,6 +70,8 @@ class GlobalContext: else: logger.warning("The output_path exists, the data will be covered.") + self._setting[GradConst.TIME_STAMP] = str(int(time.time())) + def get_context(self, key: str): if key not in self._setting: logger.warning(f"Unrecognized {key}.") diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index 8a154f4d65f63e55f6b0cf3165d3c905bcb68546..c46d55b7b481bc89a56f2eac997c1618fb2cdda2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -111,7 +111,8 @@ class CSVGenerator(Process): output_path = context.get_context(GradConst.OUTPUT_PATH) self.level = context.get_context(GradConst.LEVEL) self.bounds = context.get_context(GradConst.BOUNDS) - self.dump_dir = f"{output_path}/rank{rank_id}/Dump/" + time_stamp = context.get_context(GradConst.TIME_STAMP) + self.dump_dir = f"{output_path}/rank{rank_id}/Dump{time_stamp}/" self.save_dir = f"{output_path}/rank{rank_id}/" self.current_step = None self.stop_event = multiprocessing.Event() diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py index 1aa9fcfad10815d5845de66ab0ea6d4d7211741f..36857636fa301db37ae4267f8e18d41d9f0328a5 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py @@ -49,12 +49,10 @@ class HookInput: self.param_list = grad_context.get_context(GradConst.PARAM_LIST) self.rank_id = get_rank_id() output_path = grad_context.get_context(GradConst.OUTPUT_PATH) - self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", "Dump") + time_stamp = grad_context.get_context(GradConst.TIME_STAMP) + self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", f"Dump{time_stamp}") self.save_dir = os.path.join(output_path, f"rank{self.rank_id}") self.step_finish_flag = os.path.join(self.dump_dir, GradConst.STEP_FINISH) - if os.path.exists(self.save_dir): - logger.warning(f"Delete existing path {self.save_dir}.") - remove_path(self.save_dir) self.level = grad_context.get_context(GradConst.LEVEL) self.bounds = grad_context.get_context(GradConst.BOUNDS) self.mode = mindspore.get_context("mode") diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/anomaly_analyse.py b/debug/accuracy_tools/msprobe/mindspore/monitor/anomaly_analyse.py new file mode 100644 index 0000000000000000000000000000000000000000..d9331d2ba9e2f8ae16d33a7daa5b0335faf39e9c --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/anomaly_analyse.py @@ -0,0 +1,63 @@ +# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from msprobe.core.common.log import logger +from msprobe.core.common.const import MonitorConst +from msprobe.core.common.file_utils import save_json, create_directory, remove_path, \ + check_file_or_directory_path, load_json + + +class AnomalyDataWriter: + """ + 异常数据写入类，负责将异常数据写入到JSON文件中。 + """ + + def __init__(self, dump_path, rank) -> None: + self.dump_path = dump_path + self.dump_rank_dir = os.path.join(self.dump_path, f"rank{rank}") + self.json_path = os.path.join(self.dump_rank_dir, MonitorConst.ANOMALY_JSON) + + @staticmethod + def get_anomaly_dict(anomalies): + """将GradAnomalyData列表转换为json""" + anomalies_json = {} + for anomaly in anomalies: + anomalies_json.update({anomaly.get_key(): anomaly.to_dict()}) + return anomalies_json + + def init_detected_json(self): + """初始化落盘文件""" + create_directory(self.dump_rank_dir) + + if os.path.exists(self.json_path): + check_file_or_directory_path(self.json_path, isdir=False) + logger.warning(f"The existing file will be deleted: {self.json_path}.") + remove_path(self.json_path) + save_json(self.json_path, {}, indent=1) + + def write_detected_json(self, anomalies): + """ + 落盘异常数据 + Args: + anomalies: GradAnomalyData对象列表 + """ + anomalies_json = self.get_anomaly_dict(anomalies) + logger.info(f"{MonitorConst.ANOMALY_JSON} is at {self.dump_rank_dir}.") + + data_to_write = load_json(self.json_path) if os.path.exists(self.json_path) else {} + data_to_write.update(anomalies_json) + save_json(self.json_path, data_to_write, indent=1) diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/anomaly_detect.py b/debug/accuracy_tools/msprobe/mindspore/monitor/anomaly_detect.py index 3544ebbd025614349585bc799b15e00a5c2c7956..0211d446d3f29d02c7ad20ed92ea1aa829928686 100644 --- a/debug/accuracy_tools/msprobe/mindspore/monitor/anomaly_detect.py +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/anomaly_detect.py @@ -16,6 +16,7 @@ import itertools import os import sys +import math import statistics as st from abc import ABC from dataclasses import dataclass, field @@ -25,6 +26,7 @@ from collections import defaultdict import pandas as pd from mindspore import ops +from mindspore import Tensor from mindspore import _no_grad from msprobe.core.common.log import logger from msprobe.core.common.file_utils import change_mode, create_directory, write_df_to_csv @@ -34,7 +36,7 @@ from msprobe.core.common.const import FileCheckConst, MonitorConst class ScanRule(ABC): name = "ScanRule" - def apply(self, history, cur): + def apply(self, cur, history=None): raise NotImplementedError("abstract method apply is not implemented") @@ -44,14 +46,25 @@ class AnomalyTurbulence(ScanRule): def __init__(self, threshold) -> None: self.threshold = threshold - def apply(self, history, cur): + def apply(self, cur, history=None): + """ + :param cur: float, current metric value + :param history: float, history weighted average + :return: bool, whether the current value deviates from the historical average value of current metric + """ baseline = st.mean(history) if isinstance(history, list) else history + up_bound = baseline * (1 + self.threshold) + return abs(cur) > up_bound - up_bound = baseline + baseline * self.threshold - if baseline > 0: - return cur > up_bound - else: - return cur < up_bound + +class AnomalyNan(ScanRule): + name = "AnomalyNan" + + def __init__(self, threshold=None) -> None: + self.threshold = threshold + + def apply(self, cur, history=None): + return math.isnan(cur) or (self.threshold is not None and abs(cur) > self.threshold) class AnomalyScanner: @@ -70,7 +83,7 @@ class AnomalyScanner: rule_args = spec.get("args") # 检查必要的键是否存在 - if rule_cls_name is None or rule_args is None: + if rule_cls_name is None or (rule_cls_name == "AnomalyTurbulence" and rule_args is None): logger.warning(f"Spec is missing required keys: {spec}") continue @@ -82,7 +95,7 @@ class AnomalyScanner: continue try: - rule_instance = rule_cls(**rule_args) + rule_instance = rule_cls(**rule_args) if rule_args is not None else rule_cls() alert_rules.append(rule_instance) except Exception as e: logger.error(f"Error creating instance of rule '{rule_cls_name}': {e}") @@ -94,7 +107,7 @@ class AnomalyScanner: def scan(scan_rules: List[ScanRule], history, cur): anomaly = False for rule in scan_rules: - anomaly = rule.apply(history, cur) + anomaly = rule.apply(cur, history=history) if anomaly: return anomaly, rule.name return anomaly, None @@ -162,9 +175,8 @@ class TrainStage: OPTIMIZER_STAGE = 2 -FORWARD_KEY = [MonitorConst.ACTV_IN, MonitorConst.ACTV_OUT] -BACKWARD_KEY = [MonitorConst.ACTVGRAD_IN, MonitorConst.ACTVGRAD_OUT, - MonitorConst.PRE_GRAD, MonitorConst.POST_GRAD, MonitorConst.ACC_GRAD] +FORWARD_KEY = [MonitorConst.ACTV] +BACKWARD_KEY = [MonitorConst.ACTVGRAD, MonitorConst.PRE_GRAD, MonitorConst.POST_GRAD, MonitorConst.ACC_GRAD] OPTIMIZER_KEY = [MonitorConst.EXP_AVG, MonitorConst.EXP_AVG_SQ] TRAIN_STAGE = { **{key_: TrainStage.FORWARD_STAGE for key_ in FORWARD_KEY}, @@ -222,7 +234,7 @@ class GradAnomalyData: @staticmethod def get_train_stage(tag_name): """ - :param tag_name: "0:fc2_0/rank0/input", "0:fc1.weight/rank0/post_grad", "0:fc2.weight/rank0/exp_avg_sq" + :param tag_name: "0:fc2.input:0/rank0/actv", "0:fc1.weight/rank0/post_grad", "0:fc2.weight/rank0/exp_avg_sq" :return: int, if forward return 0; if backward return 1; if optimizer return 2 """ key_ = tag_name.split("/")[-1] @@ -254,6 +266,41 @@ class BaseWriterWithAD: self.anomaly_factory = writer_input.anomaly_factory self.anomalies = [] self.ndigits = writer_input.ndigits + self.beta = 0.99 + + @staticmethod + def stack_tensors(tensor_list): + """ + Torch not support stack cpu and xpu tensors. Group the tensors into cpu_group and xpu_group, + stack them separately, migrate xpu_group to cpu, and then restore in the order of input. + + :param tensor_list: [tensor(-1.6165), tensor(-1.0985), tensor(-1.7777), tensor(-1.8408, device='npu:0')] + :return: result: list of float + """ + cpu_tensors = [] + xpu_tensors = [] + + for tensor in tensor_list: + if isinstance(tensor, Tensor): + # 将device上的tensor先stack后to cpu + xpu_tensors.append(tensor) + else: + cpu_tensors.append(tensor) + + xpu_stack = ops.stack(xpu_tensors).tolist() if xpu_tensors else ops.tensor([]) + + # 按照输入的顺序恢复 + result = [] + cpu_tensors_idx, xpu_tensors_idx = 0, 0 + for tensor in tensor_list: + if isinstance(tensor, Tensor): + result.append(xpu_stack[xpu_tensors_idx]) + xpu_tensors_idx += 1 + else: + result.append(cpu_tensors[cpu_tensors_idx]) + cpu_tensors_idx += 1 + + return result def get_anomalies(self): """返回已检测到的异常列表 @@ -272,12 +319,17 @@ class BaseWriterWithAD: Returns: None """ - detected = False - if self.ad_rules: - avg = self._update_tag2scalars(tag, scalar_value) - detected, rule_name = self._ad(scalar_value, history=avg) + if not self.ad_rules or tag[-1] in ["shape", "dtype"]: + return + if isinstance(scalar_value, Tensor): + scalar_value = scalar_value.item() + avg = self._update_tag2scalars(tag, scalar_value) + detected, rule_name = self._ad(scalar_value, history=avg) if detected: - exception_message = f"Rule {rule_name} reports anomaly signal in {tag} at step {global_step}." + if rule_name == AnomalyTurbulence.name and tag[-1] not in ["norm", "mean"]: + return + exception_message = (f"Rule {rule_name} reports anomaly signal in {tag} at step {global_step}, " + f"current value {scalar_value}, history mean {avg}.") logger.info(f"{BCOLORS.WARNING}> {exception_message}{BCOLORS.ENDC}") # append to self.anomalies for dump if self.anomaly_factory: @@ -290,8 +342,12 @@ class BaseWriterWithAD: tags = list(itertools.product(metric_value.keys(), op_list)) for op2tensor in metric_value.values(): tensors.extend(op2tensor.values()) + + if not tensors: + return + with _no_grad(): - metric_list = ops.stack(tensors).tolist() if tensors else [] + metric_list = self.stack_tensors(tensors) for tag, metric in zip(tags, metric_list): self.add_scalar(tag, metric, step, need_explain) @@ -311,11 +367,11 @@ class BaseWriterWithAD: Returns: float: The average value before update. """ + abs_scalar_value = abs(scalar_value) if tag not in self.tag2scalars: - self.tag2scalars[tag] = {'avg': scalar_value, 'count': 0} + self.tag2scalars[tag] = {'avg': abs_scalar_value, 'count': 0} avg = self.tag2scalars[tag]['avg'] - new_avg = (avg * self.tag2scalars[tag]['count'] + scalar_value) / (self.tag2scalars[tag]['count'] + 1) - self.tag2scalars[tag]['avg'] = new_avg + self.tag2scalars[tag]['avg'] = self.beta * avg + (1 - self.beta) * abs_scalar_value self.tag2scalars[tag]['count'] += 1 return avg @@ -353,11 +409,10 @@ class CSVWriterWithAD(BaseWriterWithAD): new_data = [] for name, metric_value in self.context_dict.items(): - if MonitorConst.NAME_SEP not in name: - new_data.append([name] + [step] + metric_value) - else: - new_data.append(name.split(MonitorConst.NAME_SEP) + [step] + metric_value) - new_data = pd.DataFrame(new_data).round(self.ndigits) + new_line = name.split(MonitorConst.NAME_SEP) + metric_value + new_line.insert(2, step) + new_data.append(new_line) + new_data = pd.DataFrame(new_data).round(self.ndigits).fillna("nan") write_df_to_csv(new_data, filepath, mode='a+', header=False) self.context_dict = defaultdict(list) @@ -379,26 +434,11 @@ class CSVWriterWithAD(BaseWriterWithAD): need_explain = prefix == 'other' super().write_metrics(op_list, metric_value, step, prefix='', need_explain=need_explain) - # generate csv headers - # set hashmap to reduce the number of headers generated. - # 前向的norm用input.ops_和output.ops_，反向的用input_grad.ops_和output_grad.ops_ - if prefix in {"actv", "actv_grad"}: - if prefix == "actv": - input_and_output = [MonitorConst.ACTV_IN, MonitorConst.ACTV_OUT] - else: - input_and_output = [MonitorConst.ACTVGRAD_IN, MonitorConst.ACTVGRAD_OUT] - ops_ = [MonitorConst.DOT.join(i) for i in itertools.product(input_and_output, op_list)] - csv_header = ["module_name", "step", *ops_] + if prefix in [MonitorConst.ACTV, MonitorConst.ACTVGRAD]: + self.header = MonitorConst.CSV_HEADER_XY + op_list else: - csv_header = ["param_name", "step", *op_list] - - keys = list(metric_value.keys()) - if keys and MonitorConst.NAME_SEP in keys[0]: - csv_header.insert(0, "vpp_stage") - - self.header = csv_header + self.header = MonitorConst.CSV_HEADER + op_list self.write_csv(prefix, step) - self.header = [] def close(self): pass diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/common_func.py b/debug/accuracy_tools/msprobe/mindspore/monitor/common_func.py new file mode 100644 index 0000000000000000000000000000000000000000..ef72a75ca246a8943bf580ba490465d2cca2c09b --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/common_func.py @@ -0,0 +1,91 @@ +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from mindspore import nn +from mindspore import communication +from msprobe.mindspore.monitor.utils import logger +from msprobe.mindspore.common.utils import is_mindtorch +if is_mindtorch(): + import torch + + +def is_valid_instance(model): + return isinstance(model, torch.nn.Module) if is_mindtorch() else isinstance(model, nn.Cell) + + +def get_submodules(model): + if not is_valid_instance(model): + logger.info("Counter invalid model, nothing to hook") + return {} + return model.named_modules() if is_mindtorch() else model.cells_and_names() + + +def get_parameters(model): + if not is_valid_instance(model): + return {} + if is_mindtorch(): + return model.named_parameters() + else: + return model.parameters_and_names() + + +def get_rank(): + if comm_is_initialized(): + return communication.get_rank() + return 0 + + +def comm_is_initialized(): + return communication.GlobalComm.INITED + + +def optimizer_pre_hook(optimizer, fn): + """ + fn should be fn(optimizer, args, **kwargs) + """ + if is_mindtorch(): + origin_api = optimizer.__class__.step + + def patch_step(func, optimizer): + def wrapper(*args, **kwargs): + fn(optimizer, args, kwargs) + out = func(*args, **kwargs) + return out + return wrapper + optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) + return (optimizer.__class__.step, origin_api) + + else: + handle = optimizer.register_forward_pre_hook(fn) + return handle + + +def optimizer_post_hook(optimizer, fn): + if is_mindtorch(): + origin_api = optimizer.__class__.step + + def patch_step(func, optimizer): + def wrapper(*args, **kwargs): + out = func(*args, **kwargs) + fn(optimizer, args, kwargs) + return out + return wrapper + optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) + return (optimizer.__class__.step, origin_api) + + else: + handle = optimizer.register_forward_hook(fn) + return handle diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/features.py b/debug/accuracy_tools/msprobe/mindspore/monitor/features.py index be958dadfe8fcc50f26f16c93b3a090269235d1e..997e39f1ecb6346fcbcd33c5bcf63d9c6731ec63 100644 --- a/debug/accuracy_tools/msprobe/mindspore/monitor/features.py +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/features.py @@ -46,6 +46,8 @@ def get_max(x: Tensor): @_no_grad() def get_zeros(x: Tensor, eps: float): + if x.numel() == 0: + return Tensor(float('nan')) return mint.sum(mint.abs(x) < eps) / x.numel() @@ -54,10 +56,20 @@ def get_nans(t): return ops.isnan(t.astype(mstype.float32)).sum() +def get_shape(t): + return t.shape + + +def get_dtype(t): + return t.dtype + + FUNC_MAP = {"min" : get_min, "max" : get_max, "mean" : get_mean, "norm" : get_norm, "nans" : get_nans, - "zeros": get_zeros + "zeros": get_zeros, + "shape": get_shape, + "dtype": get_dtype } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/module_hook.py b/debug/accuracy_tools/msprobe/mindspore/monitor/module_hook.py index 068be9ff6c782bec2bf637999ef5f0eabe0c2675..474bc311f48bd55874d09d82cb699f10c2f1bfe7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/monitor/module_hook.py +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/module_hook.py @@ -20,21 +20,24 @@ from collections import defaultdict from datetime import datetime import pytz -import mindspore as ms +import pandas as pd from mindspore import Tensor, mint from mindspore import nn, _no_grad -from mindspore.communication import get_rank from msprobe.core.common.log import logger -from msprobe.core.common.const import MonitorConst +from msprobe.core.common.const import MonitorConst, Const from msprobe.core.common.file_utils import load_json, save_json +from msprobe.mindspore.common.utils import is_mindtorch +from msprobe.mindspore.monitor.common_func import is_valid_instance, get_parameters, get_submodules, get_rank from msprobe.mindspore.monitor.utils import get_summary_writer_tag_name, validate_config, step_accumulates_one, \ - is_skip_step, get_metrics, get_single_metrics, get_target_output_dir -from msprobe.mindspore.monitor.module_spec_verifier import validate_config_spec + is_skip_step, get_metrics, get_target_output_dir +from msprobe.mindspore.monitor.optimizer_collect import OptimizerMonFactory from msprobe.mindspore.monitor.anomaly_detect import AnomalyScanner, AnomalyDataFactory, \ CSVWriterWithAD, BaseWriterWithAD, WriterInput -from msprobe.mindspore.monitor.distributed.wrap_distributed import api_register, create_hooks, op_aggregate, \ - get_process_group +from msprobe.mindspore.monitor.anomaly_analyse import AnomalyDataWriter +from msprobe.mindspore.monitor.distributed.wrap_distributed import api_register, create_hooks, op_aggregate +from msprobe.core.common.file_utils import write_df_to_csv +from msprobe.core.common.utils import analyze_api_call_stack FORMAT_MAPPING = { MonitorConst.CSV: CSVWriterWithAD, @@ -88,24 +91,7 @@ class ModuleHookContext: self.actvgrad = [] self.module_name = module_name self.struct = {} - self.format_by_arg = {} - self.verified = False - self.focused_in_col = 0 - self.focused_out_col = 0 - self.ignore_in = False # no need to care when no key 'input' or 'input_grad' found - - def set_format_by_arg(self, key_name: str, target_config: dict): - cared = target_config.get(self.module_name, self.struct) - if key_name in cared: - if isinstance(cared[key_name], dict): - # current cared is self.struct - config = cared[key_name].get('config') - self.format_by_arg[key_name] = config - else: - # current cared is target_config[self.module_name] - self.format_by_arg[key_name] = cared[key_name] - elif key_name in ['input', 'input_grad']: - self.ignore_in = True + self.stack = "" def reset(self): self.actv.clear() @@ -186,6 +172,7 @@ class TrainerMon: self.config_file_path = config_file_path self.process_group = process_group self.params_have_main_grad = params_have_main_grad + self.is_mindtorch = is_mindtorch() self.config_timestamp = 0 # 后面有校验时间戳, 首次监控无需为了更新config文件时间戳而去改, 可通过dynamic_on开关直接打开 self.config = load_json(config_file_path) validate_config(self.config) @@ -218,6 +205,7 @@ class TrainerMon: self.dp_group = None self.tp_group = None self.micro_batch_number = 1 + self.optimizer_mon = None # TYPE3: 会随着训练中途config配置更新或监控状态改变而重置的变量 self.module_fwd_hook_context_by_module = defaultdict(ModuleHookContext) @@ -240,6 +228,8 @@ class TrainerMon: self.optimizer_hooked = False self.param_registered = False self.struct_printed = False + self.pre_step_hooks = [] + self.post_step_hooks = [] # 动静态区分 self.dynamic_enable = os.getenv("DYNAMIC_MONITOR", 'False').lower() == 'true' @@ -276,6 +266,7 @@ class TrainerMon: self.param_distribution = self.config.get("param_distribution", False) self.mg_direction = self.config.get('mg_direction', False) # main grad direction self.cc_distribution = self.config.get("cc_distribution", {}) # communication ops + self.stack_info = self.config.get('stack_info', False) if not self.cc_distribution.get('enable', False): self.cc_log_only = False else: @@ -283,8 +274,6 @@ class TrainerMon: self.cc_log_only = self.cc_distribution.get('cc_log_only', False) self.cc_logged_stack = defaultdict(set) self.cc_pre_hook = self.cc_distribution.get('cc_pre_hook', False) - self.handles['cc'] = api_register.initialize_hook(*create_hooks(context=self.cc_context, monitor=self)) - api_register.redirect_api() self.common_info() # 初始化AnomalyData工厂 @@ -298,18 +287,25 @@ class TrainerMon: if self.format not in FORMAT_MAPPING: logger.error(f"Unsupported format: {self.format}, use default format: {MonitorConst.CSV}") self.format = MonitorConst.CSV - writer = FORMAT_MAPPING[self.format] self.step_count_per_record = self.config.get('step_count_per_record', 1) - self.summary_writer = writer( - WriterInput( - self.tensorboard_dir, - self.alert_rules, - self.unique_id, - self.anomaly_data_factory, - self.ndigits, - self.step_count_per_record + if not self.module_rank_list or (self.rank in self.module_rank_list): + writer = FORMAT_MAPPING[self.format] + self.summary_writer = writer( + WriterInput( + self.tensorboard_dir, + self.alert_rules, + self.unique_id, + self.anomaly_data_factory, + self.ndigits, + self.step_count_per_record + ) ) - ) + + # 初始化anomaly detected文件目录 + if self.anomaly_data_factory: + self.anomaly_data_writer = AnomalyDataWriter(os.path.join(self.output_base_dir, "anomaly_detected"), + self.rank) + self.anomaly_data_writer.init_detected_json() def common_info(self): if not self.xy_distribution: @@ -341,6 +337,7 @@ class TrainerMon: self.micro_batch_number = grad_acc_steps self.dp_group = dp_group self.tp_group = tp_group + self.optimizer_mon = OptimizerMonFactory.create_optimizer_mon(optimizer) self.hook_step_final(optimizer) if not isinstance(model, list): model = [model] @@ -361,16 +358,28 @@ class TrainerMon: context.step - self.start_step) % self.step_interval == 0) if module_rank_valid and step_condition: self.has_collect_times += 1 + + if self.anomaly_data_factory: + self.anomaly_data_factory.set_call_id(self.param_name_call_id) self.write_xy_tb(context.step) self.write_grad_tb(context.step) self.write_mv_tb(context) self.write_param_tb(context) + if self.stack_info: + self.write_stack_info() + self.stack_info = False + for handle in self.handles["stack"]: + handle.remove() + self.handles["stack"].clear() if context.metric_dict: self.summary_writer.write_metrics(self.ops, context.metric_dict, context.step, 'other') context.metric_dict.clear() + if self.anomaly_data_factory: + self.anomaly_data_writer.write_detected_json(self.summary_writer.get_anomalies()) self.summary_writer.clear_anomalies() + self.call_id = 0 self.param_name_call_id.clear() @@ -380,7 +389,23 @@ class TrainerMon: context.step += 1 self.dynamic_monitor(optimizer) - optimizer.register_forward_hook(step_final_hook) + + def patch_step(func, optimizer): + def wrapper(*args, **kwargs): + for hook in self.pre_step_hooks: + hook(optimizer, args, kwargs) + out = func(*args, **kwargs) + for hook in self.post_step_hooks: + hook(optimizer, args, kwargs) + step_final_hook(optimizer, args, kwargs) + return out + return wrapper + + if self.is_mindtorch: + optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) + else: + optimizer.__class__.construct = patch_step(optimizer.__class__.construct, optimizer) + return def dynamic_monitor(self, optimizer): @@ -408,13 +433,14 @@ class TrainerMon: validate_config(config) self.config = config self.set_config() + self.start_step = context.step # 动态启停时不受原start_step影响，永远从下一步开始 logger.warning(f"config is updated at step{context.step - 1}, " f"will start new hook at step{context.step}.") except Exception as e: logger.error(f"set config wrong because {e}, not updated, please check!!!") return - self._remove_all_hooks() + self._remove_all_hooks(optimizer) self.register_hooks(optimizer) def register_hooks(self, optimizer): @@ -422,6 +448,9 @@ class TrainerMon: self.hook_modules() self.hook_optimizer(optimizer) self._patch_grad_sync() + if self.cc_distribution.get('enable', False): + self.handles['cc'] = api_register.initialize_hook(*create_hooks(context=self.cc_context, monitor=self)) + api_register.redirect_api() self.monitoring = True def hook_modules(self): @@ -436,45 +465,36 @@ class TrainerMon: hooked_count = 0 for vpp_stage, model_chunk in enumerate(self.model): - if not isinstance(model_chunk, nn.Cell): + if not is_valid_instance(model_chunk): logger.info("Target Model is not Cell") continue vpp_stage = f'{vpp_stage}{MonitorConst.NAME_SEP}' - targets = [x for x, _ in model_chunk.cells_and_names()] if self.print_struct else self.targets.keys() + targets = [x for x, _ in get_submodules(model_chunk)] if self.print_struct else self.targets.keys() hooked_count += self._hook_module(targets, model_chunk, vpp_stage) logger.info(f"> {hooked_count} modules are monitored.") def hook_optimizer(self, optimizer): - def optimizer_pre_hook_function(opt, grad_names, gradients): + def optimizer_pre_step_hook(opt, *args, **kwargs): context = self.optimizer_context[opt] if is_skip_step(context.step, self.start_step, self.step_interval, self.has_collect_times, self.collect_times): return - gradient_list = gradients[0] if isinstance(gradients, tuple) else gradients - is_select = self.is_select - for idx, grad in enumerate(gradient_list): - grad_name = grad_names[idx] - if is_select and grad_name not in self.targets: - continue - get_single_metrics(self.ops, grad_name, grad, context.param_weight_grad) - - if self.mv_distribution: - # fetch mean - for param in m_list: - name = param.name - if is_select and name not in self.targets: - continue - get_single_metrics(self.ops, name, param, context.exp_avg_metric) - # fetch variance - for param in v_list: - name = param.name - if is_select and name not in self.targets: - continue - get_single_metrics(self.ops, name, param, context.exp_avg_sq_metric) - if self.param_distribution: - for param in param_list: - get_single_metrics(self.ops, param.name, param, context.param_metric) - self.generate_wgrad_metrics() + + grad_dict = {} + if self.wg_distribution: + grad_dict = self.optimizer_mon.fetch_grad(self, self.param2name) + + if self.mv_distribution or self.ur_distribution or self.mg_direction: + if self.is_mindtorch: + context.param_exp_avg, context.param_exp_avg_sq, context.param_adam_update, \ + context.param_adam_ratio = self.optimizer_mon.fetch_mv(self, self.param2name) + else: + context.param_exp_avg, context.param_exp_avg_sq = self.get_mv_for_ms(optimizer) + + self.generate_wgrad_metrics(grad_dict) + self.generate_mv_metrics(context) + self.generate_param_metrics(context, MonitorConst.PRE_PARAM) + metric_dict = {} for cc in self.cc_context.values(): cc.aggregate() @@ -486,63 +506,88 @@ class TrainerMon: context.metric_dict = metric_dict return - def optimizer_pre_hook_wrapper(func, grad_names): - def wrapper(opt, gradients): - return func(opt, grad_names, gradients) - return wrapper + def optimizer_post_step_hook(optimizer, args, kwargs): + context = self.optimizer_context[optimizer] + self.generate_param_metrics(context, MonitorConst.POST_PARAM) + if self.optimizer_hooked or not self.is_target_rank(): return - m_list = [] - v_list = [] - param_list = [] - grad_names = [] - for param in optimizer.get_parameters(): - if MonitorConst.EXP_AVG_SQ in param.name: - v_list.append(param) - elif MonitorConst.EXP_AVG in param.name: - m_list.append(param) - elif param.name in ['global_step', 'learning_rate']: - pass - else: - param_list.append(param) - grad_names.append(param.name) - - handle = optimizer.register_forward_pre_hook( - optimizer_pre_hook_wrapper(optimizer_pre_hook_function, grad_names)) - self.handles['optimizer'].append(handle) + self.pre_step_hooks.append(optimizer_pre_step_hook) + self.post_step_hooks.append(optimizer_post_step_hook) self.optimizer_hooked = True return - def generate_wgrad_metrics(self): + def generate_wgrad_metrics(self, grad_dict): if not self.wg_distribution: - return {}, {} + return if self.weight_hooked: - try: - get_metrics(self.ops, self.grad_context.acc, self.eps, self.grad_context.acc_metric) - except Exception as e: - logger.warning(f"An error occurred while generating wgrad pre metrics") - return {}, {} + get_metrics(self.ops, self.grad_context.acc, self.eps, self.grad_context.acc_metric) - grad_dict = {} - for param, name in self.param2name.items(): - if self.duplicate_param.get(name, False): - continue - grad = param.main_grad if self.params_have_main_grad else param.grad - if grad is None: - logger.warning(f"grad is None: {name}, maybe something wrong happened.") + get_metrics(self.ops, grad_dict, self.eps, self.grad_context.post) + + def generate_param_map(self, tag, param_tensor): + metrics = {} + if not self.is_mindtorch: + return param_tensor + for name in self.param2name.values(): + key = get_summary_writer_tag_name(name, tag, self.rank) + self.register_param_call_id("optimizer_pre_step_hook", key) + if name not in param_tensor or param_tensor[name] is None: continue - tag = self.name2tag.get(name, {}).get(MonitorConst.POST_GRAD) - self._register_param_call_id("hook_optimizer", tag) - grad_dict[tag] = grad - try: - get_metrics(self.ops, grad_dict, self.eps, self.grad_context.post) - except Exception as e: - logger.warning(f"An error occurred while generating wgrad post metrics") + metrics[key] = param_tensor[name] + return metrics + + def generate_param_metrics(self, opt_context, stage=MonitorConst.PRE_PARAM): + if not self.param_distribution: + return + tag2param = { + self.name2tag.get(name, {}).get(stage): param + for name, param in self.name2param.items() + if param.numel() != 0 + } + get_metrics(self.ops, tag2param, self.eps, opt_context.param_metric) + + def get_mv_for_ms(self, opt): + if not self.mv_distribution: return {}, {} - return self.grad_context.post, self.grad_context.pre + common_opt = opt + if not is_valid_instance(opt): + common_opt = getattr(opt, 'optimizer') + if not is_valid_instance(common_opt): + logger.warning("Optimizer is not valid, please check usage") + return {}, {} + m_dict = {} + v_dict = {} + for name, param in get_parameters(common_opt): + if MonitorConst.EXP_AVG_SQ in name: + m_dict[name] = param + elif MonitorConst.EXP_AVG in name: + v_dict[name] = param + return m_dict, v_dict + + def generate_mv_metrics(self, opt_context): + if not self.mv_distribution: + return + opt_context.exp_avg_metric = {} + opt_context.exp_avg_sq_metric = {} + m_tag_tensor_map = self.generate_param_map(MonitorConst.EXP_AVG, opt_context.param_exp_avg) + v_tag_tensor_map = self.generate_param_map(MonitorConst.EXP_AVG_SQ, opt_context.param_exp_avg_sq) + get_metrics(self.ops, m_tag_tensor_map, self.eps, opt_context.exp_avg_metric) + get_metrics(self.ops, v_tag_tensor_map, self.eps, opt_context.exp_avg_sq_metric) + + def write_stack_info(self): + stack_data = [] + header = ["module_name", "stack_info"] + stack_data.append(header) + for _, fwd_context in self.module_fwd_hook_context_by_module.items(): + stack_data.append([fwd_context.module_name, fwd_context.stack]) + filepath = os.path.join(self.tensorboard_dir, f'stack_info.csv') + if not os.path.exists(filepath): + data_frame = pd.DataFrame(columns=stack_data) + write_df_to_csv(data_frame, filepath) def write_xy_tb(self, step): if not self.xy_distribution: @@ -550,21 +595,25 @@ class TrainerMon: for _, fwd_context in self.module_fwd_hook_context_by_module.items(): if len(fwd_context.actv) == 0: continue - self.summary_writer.write_metrics(self.ops, fwd_context.actv, step, 'actv') + self.summary_writer.write_metrics(self.ops, fwd_context.actv, step, MonitorConst.ACTV) fwd_context.actv.clear() if self.grad_context.actv: - self.summary_writer.write_metrics(self.ops, self.grad_context.actv, step, 'actv_grad') + self.summary_writer.write_metrics(self.ops, self.grad_context.actv, step, MonitorConst.ACTVGRAD) def write_param_tb(self, opt_context): if not self.param_distribution: return - self.summary_writer.write_metrics(self.ops, opt_context.param_metric, opt_context.step, 'param') + param_metrics = {k: v for k, v in opt_context.param_metric.items() if MonitorConst.PRE_PARAM in k} + updated_param_metrics = {k: v for k, v in opt_context.param_metric.items() if MonitorConst.POST_PARAM in k} + self.summary_writer.write_metrics(self.ops, param_metrics, opt_context.step, MonitorConst.PRE_PARAM) + self.summary_writer.write_metrics(self.ops, updated_param_metrics, opt_context.step, MonitorConst.POST_PARAM) def write_mv_tb(self, opt_context): if not self.mv_distribution: return - self.summary_writer.write_metrics(self.ops, opt_context.exp_avg_metric, opt_context.step, 'exp_avg') - self.summary_writer.write_metrics(self.ops, opt_context.exp_avg_sq_metric, opt_context.step, 'exp_avg_sq') + self.summary_writer.write_metrics(self.ops, opt_context.exp_avg_metric, opt_context.step, MonitorConst.EXP_AVG) + self.summary_writer.write_metrics(self.ops, opt_context.exp_avg_sq_metric, opt_context.step, + MonitorConst.EXP_AVG_SQ) def write_grad_tb(self, step): if not self.wg_distribution: @@ -578,13 +627,38 @@ class TrainerMon: return False return True - def build_tbtag_tensor_map(self, module_name, tag, tensor): - metrics = {} - key = get_summary_writer_tag_name(module_name, tag, str(self.rank)) + def build_tbtag_tensor_map(self, module_name, suffix, tag, tensor): + """ + :param module_name: str of module name + :param suffix: + :param tag: + :param tensor: torch.tensor or tuple/list of torch.tensor + :return: tensor_map + """ + tensor_map = {} if isinstance(tensor, Tensor): - self._register_param_call_id("_hook_module", key) - metrics[key] = tensor - return metrics + tensor = [tensor] + if isinstance(tensor, tuple) or isinstance(tensor, list): + if len(tensor) == 1: + key = get_summary_writer_tag_name(module_name + suffix, tag, self.rank) + self.register_param_call_id("_hook_module", key) + tensor_map[key] = tensor[0] + else: + for i, tensor_i in enumerate(tensor): + key = get_summary_writer_tag_name(module_name + f"_{i}" + suffix, tag, self.rank) + self.register_param_call_id("_hook_module", key) + tensor_map[key] = tensor_i + return tensor_map + + def register_param_call_id(self, hook_name: str, key: str): + """ + :param hook_name: + :param key: str, '0:relu_0/output_grad' + :return: + """ + logger.debug(f"{hook_name} {key}: {self.call_id}") + self.param_name_call_id[key] = self.call_id + self.call_id += 1 def _register_param_name(self): for vpp_stage, model_chunk in enumerate(self.model): @@ -593,8 +667,7 @@ class TrainerMon: def _register_chunk(self, model_chunk, prefix): index = 0 - for param in model_chunk.get_parameters(): - param_name = param.name + for param_name, param in get_parameters(model_chunk): if not param.requires_grad: continue if self._is_target_param(param_name, param, prefix): @@ -609,25 +682,37 @@ class TrainerMon: self.duplicate_param[name] = True if self.dp_group and param_is_data_parallel_duplicate(self.dp_group): self.duplicate_param[name] = True + keywords = [ + MonitorConst.PRE_GRAD, + MonitorConst.POST_GRAD, + MonitorConst.PRE_PARAM, + MonitorConst.POST_PARAM + ] self.name2tag[name] = { - MonitorConst.PRE_GRAD: get_summary_writer_tag_name(name, MonitorConst.PRE_GRAD, self.rank), - MonitorConst.POST_GRAD: get_summary_writer_tag_name(name, MonitorConst.POST_GRAD, self.rank) + k: get_summary_writer_tag_name(name, k, self.rank) + for k in keywords } index += 1 def _hook_module(self, target_names, module, vpp_stage=''): - if not isinstance(module, nn.Cell): + if not is_valid_instance(module): # nothing to hook return 0 - def fwd_hook_fun(module, module_input, module_output, name): + def fwd_hook_fun(module, args, kwargs, module_output, name): + + module_input = [tensor for tensor in args if isinstance(tensor, Tensor)] + if kwargs: + kwargs_tensors = [tensor for tensor in kwargs.values() if isinstance(tensor, Tensor)] + module_input.extend(kwargs_tensors) + if module not in self.module_fwd_hook_context_by_module: self.module_fwd_hook_context_by_module[module] = ModuleHookContext(name) context: ModuleHookContext = self.module_fwd_hook_context_by_module[module] if not context.struct: context.struct = { - MonitorConst.ACTV_IN: get_param_struct(module_input), - MonitorConst.ACTV_OUT: get_param_struct(module_output) + Const.INPUT: get_param_struct(module_input), + Const.OUTPUT: get_param_struct(module_output) } if self.print_struct: self.module_struct[context.module_name].update(context.struct) @@ -638,31 +723,16 @@ class TrainerMon: self.collect_times): step_accumulates_one(context, self.micro_batch_number) return - if not context.format_by_arg: - context.set_format_by_arg(MonitorConst.ACTV_IN, self.targets) - context.set_format_by_arg(MonitorConst.ACTV_OUT, self.targets) - if not context.format_by_arg: - return - if not context.verified: - if not context.ignore_in: - context.focused_in_col = validate_config_spec(context.format_by_arg[MonitorConst.ACTV_IN], - module_input, context.module_name, - MonitorConst.ACTV_IN) - context.focused_out_col = validate_config_spec(context.format_by_arg[MonitorConst.ACTV_OUT], - module_output, context.module_name, - MonitorConst.ACTV_OUT) - context.verified = True tbtag_tensor_map = {} - if not context.ignore_in: - cared_input = module_input if context.focused_in_col is None else module_input[context.focused_in_col] - tbtag_tensor_map.update( - self.build_tbtag_tensor_map(f'{context.module_name}_{context.micro_step}', MonitorConst.ACTV_IN, - cared_input)) - cared_output = module_output if context.focused_out_col is None else module_output[context.focused_out_col] tbtag_tensor_map.update( - self.build_tbtag_tensor_map(f'{context.module_name}_{context.micro_step}', MonitorConst.ACTV_OUT, - cared_output)) + self.build_tbtag_tensor_map( + f'{context.module_name}.{Const.INPUT}', f'{MonitorConst.NAME_SEP}{context.micro_step}', + MonitorConst.ACTV, module_input)) + tbtag_tensor_map.update( + self.build_tbtag_tensor_map( + f'{context.module_name}.{Const.OUTPUT}', f'{MonitorConst.NAME_SEP}{context.micro_step}', + MonitorConst.ACTV, module_output)) try: get_metrics(self.ops, tbtag_tensor_map, self.eps, context.actv) except Exception as e: @@ -687,31 +757,16 @@ class TrainerMon: step_accumulates_one(context, self.micro_batch_number) return - if not context.format_by_arg: - context.set_format_by_arg(MonitorConst.ACTVGRAD_IN, self.targets) - context.set_format_by_arg(MonitorConst.ACTVGRAD_OUT, self.targets) - if not context.format_by_arg: - return - if not context.verified: - if not context.ignore_in: - context.focused_in_col = validate_config_spec(context.format_by_arg[MonitorConst.ACTVGRAD_IN], - input_grad, context.module_name, - MonitorConst.ACTVGRAD_IN) - context.focused_out_col = validate_config_spec(context.format_by_arg[MonitorConst.ACTVGRAD_OUT], - output_grad, context.module_name, - MonitorConst.ACTVGRAD_OUT) - context.verified = True - tbtag_tensor_map = {} - if not context.ignore_in: - cared_input_grad = input_grad if context.focused_in_col is None else input_grad[context.focused_in_col] - tbtag_tensor_map.update( - self.build_tbtag_tensor_map( - f'{context.module_name}_{context.micro_step}', MonitorConst.ACTVGRAD_IN, cared_input_grad)) - cared_output_grad = output_grad if context.focused_out_col is None else output_grad[context.focused_out_col] tbtag_tensor_map.update( - self.build_tbtag_tensor_map(f'{context.module_name}_{context.micro_step}', MonitorConst.ACTVGRAD_OUT, - cared_output_grad)) + self.build_tbtag_tensor_map( + f'{context.module_name}.{Const.INPUT}', f'{MonitorConst.NAME_SEP}{context.micro_step}', + MonitorConst.ACTVGRAD, input_grad)) + + tbtag_tensor_map.update( + self.build_tbtag_tensor_map( + f'{context.module_name}.{Const.OUTPUT}', f'{MonitorConst.NAME_SEP}{context.micro_step}', + MonitorConst.ACTVGRAD, output_grad)) if context.micro_step == 0 and context.actvgrad: logger.warning(f"actvgrad context of {context.module_name} is not empty when first micro_step, " @@ -726,20 +781,33 @@ class TrainerMon: return def fwd_hook_fun_wrapper(fwd_hook_fun, name): - def wrapper(module, module_input, module_output): - return fwd_hook_fun(module, module_input, module_output, name) + def wrapper(module, args, kwargs, module_output): + return fwd_hook_fun(module, args, kwargs, module_output, name) return wrapper + def stack_hook(module, args, kwargs, module_output, name): + if module not in self.module_fwd_hook_context_by_module: + self.module_fwd_hook_context_by_module[module] = ModuleHookContext(name) + context: ModuleHookContext = self.module_fwd_hook_context_by_module[module] + context.stack = analyze_api_call_stack(name) + return + if self.backward_only and self.forward_only: logger.warning('not enable backward_only and forward_only simultaneously') hooked_count = 0 - if self.xy_distribution or self.print_struct: - for module_name, submodule in module.cells_and_names(): - name = self._is_target_module(module_name, target_names, vpp_stage) - if not name: - continue + + for module_name, submodule in get_submodules(module): + if self.stack_info: + name = vpp_stage + squash_param_name(module_name) + handle = submodule.register_forward_hook(fwd_hook_fun_wrapper(stack_hook, name=name), with_kwargs=True) + self.handles["stack"].append(handle) + name = self._is_target_module(module_name, target_names, vpp_stage) + if not name: + continue + if self.xy_distribution or self.print_struct: if not self.backward_only: - handle = submodule.register_forward_hook(fwd_hook_fun_wrapper(fwd_hook_fun, name=name)) + handle = submodule.register_forward_hook(fwd_hook_fun_wrapper(fwd_hook_fun, name=name), + with_kwargs=True) self.handles['xy'].append(handle) if not self.forward_only: handle = submodule.register_backward_hook(bwd_hook_fun) @@ -760,7 +828,7 @@ class TrainerMon: @_no_grad() def param_hook(grad, context_dict, param, key): param.micro_step += 1 - self._register_param_call_id("param_hook", key) + self.register_param_call_id("param_hook", key) if param.micro_step == self.micro_batch_number: param.micro_step = 0 context_dict[key] = grad @@ -799,17 +867,7 @@ class TrainerMon: return pattern return "" - def _register_param_call_id(self, hook_name: str, key: str): - """ - :param hook_name: - :param key: str, '0:relu_0/output_grad' - :return: - """ - logger.debug(f"{hook_name} {key}: {self.call_id}") - self.param_name_call_id[key] = self.call_id - self.call_id += 1 - - def _remove_all_hooks(self): + def _remove_all_hooks(self, optimizer): # 清空hook handle for handle in self.handles['xy']: handle.remove() @@ -827,9 +885,8 @@ class TrainerMon: self.weight_hooked = False if self.optimizer_hooked: - for handle in self.handles['optimizer']: - handle.remove() - self.handles['optimizer'].clear() + self.pre_step_hooks.clear() + self.post_step_hooks.clear() for _, context in self.optimizer_context.items(): context.reset() self.optimizer_hooked = False @@ -837,6 +894,7 @@ class TrainerMon: for handle in self.handles['cc']: handle.remove() self.handles['cc'].clear() + api_register.restore_api() for _, context in self.cc_context.items(): context.reset() @@ -867,4 +925,4 @@ class TrainerMon: except Exception as e: logger.warning(f"Finish monitor, set config'dynamic_on=False fail because {e}, please check!!!") logger.info("Finish monitor") - self._remove_all_hooks() + self._remove_all_hooks(optimizer) diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/module_spec_verifier.py b/debug/accuracy_tools/msprobe/mindspore/monitor/module_spec_verifier.py deleted file mode 100644 index c06e8ea10f6a2178c3670e596ad64e333db44cab..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/msprobe/mindspore/monitor/module_spec_verifier.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import abc -from mindspore import Tensor - -from msprobe.core.common.log import logger - - -# 用于存储所有validator实现类的注册表 -config_validator_registry = {} - - -def register_config_validator(cls): - """装饰器 用于注册ConfigValidator的实现类""" - config_validator_registry[cls.__name__] = cls - return cls - - -class ConfigValidator(metaclass=abc.ABCMeta): - @abc.abstractmethod - def check_pattern_match(self, config_spec: str): - pass - - @abc.abstractmethod - def validate(self, actual_data, module_name: str, data_type: str, pattern_match): - pass - - -@register_config_validator -class TensorValidator(ConfigValidator): - def check_pattern_match(self, config_spec: str): - pattern = re.compile(r"tensor") - return pattern.match(config_spec) - - def validate(self, actual_data, module_name: str, data_type: str, pattern_match): - if not isinstance(actual_data, Tensor): - raise ValueError( - f"Format of {module_name} {data_type} does not match the required format 'tensor' in config.") - - -@register_config_validator -class TupleValidator(ConfigValidator): - def check_pattern_match(self, config_spec: str): - pattern = re.compile(r"tuple\[(\d+)\]:?(\d+)?") - return pattern.match(config_spec) - - def validate(self, actual_data, module_name: str, data_type: str, pattern_match): - length, index = pattern_match.groups() - if index is None: - index = 0 - length, index = int(length), int(index) - - if not (0 <= index < length): - raise ValueError( - f"Format of {module_name} {data_type} in config.json does not match the required format 'tuple[x]:y'." - f"y must be greater than or equal to 0 and less than x.") - if not isinstance(actual_data, tuple): - raise ValueError( - f"Type of {module_name} {data_type} does not match spec of config.json, should be tuple, please check.") - if len(actual_data) != length: - raise ValueError( - f"Length of {module_name} {data_type} does not match spec of config.json, should be {length}, " - f"actual is {len(actual_data)} please check.") - return index - - -def validate_config_spec(config_spec: str, actual_data, module_name: str, data_type: str): - focused_col = None - for _, validator_cls in config_validator_registry.items(): - config_validator = validator_cls() - pattern_match = config_validator.check_pattern_match(config_spec) - if pattern_match: - try: - focused_col = config_validator.validate(actual_data, module_name, data_type, pattern_match) - except ValueError as e: - logger.warning(f"config spec validate failed: {str(e)}") - return focused_col - logger.warning(f"config spec in {module_name} {data_type} not supported, " - f"expected spec:'tuple\[(\d+)\]:(\d+)' or 'tensor', actual spec: {config_spec}.") - return focused_col \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/optimizer_collect.py b/debug/accuracy_tools/msprobe/mindspore/monitor/optimizer_collect.py new file mode 100644 index 0000000000000000000000000000000000000000..c12e892e5c964a5821534c653d458ef867d0ca80 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/optimizer_collect.py @@ -0,0 +1,322 @@ +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import abstractmethod + +from mindspore import mint, ops + +from msprobe.mindspore.common.log import logger +from msprobe.core.common.const import MonitorConst + + +class OptimizerMon(object): + def __init__(self, optim) -> None: + self.fp16_to_fp32_param = {} + self.optim = optim + + def narrow_from_flatten(self, param, flatten_state): + return flatten_state + + def fetch_grad(self, monitor, params2name): + if not self.fp16_to_fp32_param: + self.map_fp16_to_fp32_param(self.optim) + + grad_dict = {} + first_param = True + for param, name in params2name.items(): + if monitor.duplicate_param.get(name, False): + continue + if self.fp16_to_fp32_param and param not in self.fp16_to_fp32_param: + continue + grad = param.main_grad if monitor.params_have_main_grad else param.grad + element_in_cur_partition = self.fp16_to_fp32_param.get(param, param).numel() + if param.numel() != element_in_cur_partition: + if first_param: + grad = grad.flatten()[-element_in_cur_partition:] + else: # supposed to be the last one + grad = grad.flatten()[:element_in_cur_partition] + first_param = False + if grad is None: + continue + tag = monitor.name2tag.get(name, {}).get(MonitorConst.POST_GRAD) + monitor.register_param_call_id("hook_optimizer", tag) + grad_dict[tag] = grad + return grad_dict + + def map_fp16_to_fp32_param(self, optim): + pass + + def fetch_mv(self, monitor, params2name): + if not self.fp16_to_fp32_param: + self.map_fp16_to_fp32_param(self.optim) + exp_avg_dict = {} + exp_avg_sq_dict = {} + update_dict = {} + ratio_dict = {} + + if hasattr(self.optim, 'state'): + state = self.optim.state + elif hasattr(self.optim, 'optimizer') and hasattr(self.optim.optimizer, 'state'): + state = self.optim.optimizer.state + else: + logger.warning('optimizer state can not accessed') + return exp_avg_dict, exp_avg_sq_dict, update_dict, ratio_dict + + for lp_param, name in params2name.items(): + if lp_param in self.fp16_to_fp32_param: + hp_param = self.fp16_to_fp32_param[lp_param] + else: + hp_param = lp_param + + if hp_param in state: + state_param = state.get(hp_param, None) + exp_avg = self.narrow_from_flatten(lp_param, state_param.get("exp_avg", None)) + exp_avg_sq = self.narrow_from_flatten(lp_param, state_param.get("exp_avg_sq", None)) + if monitor.mv_distribution: + exp_avg_dict[name] = exp_avg + exp_avg_sq_dict[name] = exp_avg_sq + if monitor.mg_direction: + exp_avg_dict[name] = exp_avg + if monitor.ur_distribution: + if len(self.optim.param_groups) > 1: + logger.info(f"the length of optimizer.param_groups is {len(self.optim.param_groups)}.") + if 'step' in state_param: + step = state_param['step'] # Optimizer from pytorch or FusedAdam from apex(used by megatron) + elif 'step' in self.optim.param_groups[0]: + step = self.optim.param_groups[0]['step'] # AdamW from mindspeed + else: + logger.warning(f"step of {name} is None, maybe something wrong happened.") + continue + exp_avg_hat = exp_avg / (1 - self.optim.defaults['betas'][0] ** step) + exp_avg_sq_hat = exp_avg_sq / (1 - self.optim.defaults['betas'][1] ** step) + update_dict[name] = exp_avg_hat / (mint.sqrt(exp_avg_sq_hat) + self.optim.defaults['eps']) + ratio_dict[name] = exp_avg_hat / mint.sqrt(exp_avg_sq_hat) + monitor.update_heatmap_visualizer[name].pre_cal(update_dict[name]) + monitor.ratio_heatmap_visualizer[name].pre_cal(ratio_dict[name]) + return exp_avg_dict, exp_avg_sq_dict, update_dict, ratio_dict + + +class MixPrecisionOptimizerMon(OptimizerMon): + """ + 混合精度优化器监控类。在混合精度训练中监控和管理优化器。 + 混合精度训练通过适当降低某些计算的精度来加速训练过程并减少内存消耗。 + """ + def map_fp16_to_fp32_param(self, optim): + for fp16_group, fp32_group in zip(optim.float16_groups, optim.fp32_from_float16_groups): + for fp16_param, fp32_param in zip(fp16_group, fp32_group): + self.fp16_to_fp32_param[fp16_param] = fp32_param + + +class MegatronDistributedOptimizerMon(OptimizerMon): + def map_fp16_to_fp32_param(self, optim): + if not (hasattr(optim, "model_float16_groups") and + hasattr(optim, "shard_fp32_from_float16_groups")): + raise Exception( + "megatron distributed optimizer should have model_float16_groups and shard_fp32_from_float16_groups, " + "if not, please check megatron-lm version") + for fp16_group, shard_fp32_group in zip(optim.model_float16_groups, + optim.shard_fp32_from_float16_groups): + for fp16_param, shard_fp32_param in zip(fp16_group, shard_fp32_group): + self.fp16_to_fp32_param[fp16_param] = shard_fp32_param + + +class MegatronChainedDistributedOptimizerMon(MegatronDistributedOptimizerMon): + def map_fp16_to_fp32_param(self, optim): + for opt in optim.chained_optimizers: + super().map_fp16_to_fp32_param(opt) + + if not hasattr(self.optim, 'state'): + optim.state = {} + for opt in self.optim.chained_optimizers: + self.optim.state.update(opt.optimizer.state) + + +class MegatronChainedMixPrecisionOptimizerMon(MixPrecisionOptimizerMon): + def map_fp16_to_fp32_param(self, optim): + for opt in optim.chained_optimizers: + super().map_fp16_to_fp32_param(opt) + + if not hasattr(self.optim, 'state'): + optim.state = {} + for opt in self.optim.chained_optimizers: + self.optim.state.update(opt.optimizer.state) + + +class DeepSpeedZeroOptimizerMon(OptimizerMon): + """ + Base monitor class for DeepSpeed ZeRO optimizer. + ZeRO stage 0 no partition + ZeRO stage 1 partitions optimizer states across data parallel processes. + ZeRO stage 2 additionally partitions gradients. + ZeRO stage 3 additionally partitions parameters. + + This class provides monitoring capabilities for ZeRO optimizers by: + - Handling gradient collection for different ZeRO stages + - Managing optimizer state access for monitoring + """ + def __init__(self, optim): + super().__init__(optim) + self.stage = '' + self.bit16_groups = [] + self.fp32_flat_groups = [] + self.param2group = () + self.param2index = [] + self.group_offset = {} + + @abstractmethod + def get_grad_for_param(self, lp_param, group_idx, param_id): + raise NotImplementedError + + def param_not_in_partition(self, lp_param, group_idx): + param_slice_mapping = self.optim.state_dict()['param_slice_mappings'][group_idx] + hp_address = param_slice_mapping.get(self.optim.param_names.get(lp_param)) + return hp_address is None + + def get_position(self, lp_param, group_idx): + param_slice_mapping = self.optim.state_dict()['param_slice_mappings'][group_idx] + hp_address = param_slice_mapping.get(self.optim.param_names.get(lp_param)) + return hp_address.start, hp_address.numel + + def get_group_index(self): + param2group = {} + for group_idx, bit16_group in enumerate(self.bit16_groups): + for param in bit16_group: + param2group[param] = group_idx + return param2group + + def get_param_index(self, lp_param, group_idx): + if not self.param2index: + for group in self.bit16_groups: + param2index = {} + for index, param in enumerate(group): + param2index[param] = index + self.param2index.append(param2index) + + return self.param2index[group_idx][lp_param] + + def narrow_from_flatten(self, param, flatten_state): + if flatten_state is None: + return flatten_state + group_idx = self.param2group[param] + if self.param_not_in_partition(param, group_idx): + return None + start, numel = self.get_position(param, group_idx) + return flatten_state.narrow(0, start, numel) + + def map_fp16_to_fp32_param(self, optim): + for group_idx, group in enumerate(self.bit16_groups): + for param in group: + self.fp16_to_fp32_param[param] = self.fp32_flat_groups[group_idx] + + def fetch_grad(self, monitor, params2name): + grad_dict = {} + for lp_param, name in params2name.items(): + group_idx = self.param2group[lp_param] + param_id = self.get_param_index(lp_param, group_idx) + if self.param_not_in_partition(lp_param, group_idx): + continue + if self.stage == '1or2': + param_id = param_id - self.group_offset[group_idx] - 1 + grad = self.get_grad_for_param(lp_param, group_idx, param_id) + tag = monitor.name2tag.get(name, {}).get(MonitorConst.POST_GRAD) + monitor.register_param_call_id("hook_optimizer", tag) + grad_dict[tag] = grad + + return grad_dict + + +class DeepSpeedZeroOptimizerStage0Mon(DeepSpeedZeroOptimizerMon): + def __init__(self, optim): + super().__init__(optim) + self.stage = '0' + self.bit16_groups = optim.bf16_groups + self.fp32_flat_groups = optim.fp32_groups_flat_partition + self.param2group = self.get_group_index() + + def get_grad_for_param(self, lp_param, group_idx, param_id): + return self.optim.fp32_groups_gradient_dict[group_idx][param_id] + + +class DeepSpeedZeroOptimizerStage1or2Mon(DeepSpeedZeroOptimizerMon): + def __init__(self, optim): + super().__init__(optim) + self.stage = '1or2' + self.bit16_groups = optim.bit16_groups + self.fp32_flat_groups = optim.single_partition_of_fp32_groups + self.param2group = self.get_group_index() + self.group_offset = {} + self.get_group_offset() + + def get_grad_for_param(self, lp_param, group_idx, param_id): + if getattr(self.optim, "cpu_offload", False): + grads = self.optim.single_partition_of_fp32_groups[group_idx].grad + start, numel = self.get_position(lp_param, group_idx) + grad = grads.narrow(0, start, numel) + else: + grad = self.optim.averaged_gradients[group_idx][param_id] + return grad + + def get_group_offset(self): + for group_idx, group in enumerate(self.bit16_groups): + self.group_offset[group_idx] = -1 + for lp_param in group: + if self.param_not_in_partition(lp_param, group_idx): + self.group_offset[group_idx] = self.get_param_index(lp_param, group_idx) + else: + break + + +class DeepSpeedZeroOptimizerStage3Mon(DeepSpeedZeroOptimizerMon): + def __init__(self, optim): + super().__init__(optim) + self.stage = '3' + self.bit16_groups = optim.fp16_groups + self.fp32_flat_groups = optim.fp32_partitioned_groups_flat + self.param2group = self.get_group_index() + + def param_not_in_partition(self, param, group_index): + """Each param partioned across all zero ranks""" + return False + + def get_position(self, lp_param, group_idx): + param_id = self.optim.get_param_id(lp_param) + return self.optim.grad_position[param_id][1:] + + def get_grad_for_param(self, lp_param, group_idx, param_id): + return self.optim.averaged_gradients[group_idx][param_id] + + +class OptimizerMonFactory: + _optimizer_mon_map = { + "FP32Optimizer": OptimizerMon, + "Float16OptimizerWithFloat16Params": MixPrecisionOptimizerMon, + "DistributedOptimizer": MegatronDistributedOptimizerMon, + "ChainedDistributedOptimizer": MegatronChainedDistributedOptimizerMon, + "ChainedFloat16OptimizerWithFloat16Params": MegatronChainedMixPrecisionOptimizerMon, + "BF16_Optimizer": DeepSpeedZeroOptimizerStage0Mon, + "DeepSpeedZeroOptimizer": DeepSpeedZeroOptimizerStage1or2Mon, + "DeepSpeedZeroOptimizer_Stage3": DeepSpeedZeroOptimizerStage3Mon, + "Adam": OptimizerMon + } + + @staticmethod + def create_optimizer_mon(optimizer): + # auto replace opt_ty + optimizer_class = optimizer.__class__.__name__ + if optimizer_class == "ChainedOptimizer": + optimizer_class = "Chained" + optimizer.chained_optimizers[0].__class__.__name__ + logger.info(f'The optimizer type is {optimizer_class}') + + optimizer_mon_class = OptimizerMonFactory._optimizer_mon_map.get(optimizer_class, OptimizerMon) + return optimizer_mon_class(optimizer) diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/utils.py b/debug/accuracy_tools/msprobe/mindspore/monitor/utils.py index c85e66a65ba26fdbc1d10a8e55c8273236409b36..d7270b1ce1331266336310a9ff3865897c7ae1b6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/monitor/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/utils.py @@ -35,7 +35,10 @@ def get_single_metrics(op_list, tag, tensor, output=None): if hasattr(statistic, "dtype") and statistic.dtype == mstype.bfloat16: statistic = float(statistic) statistic = Tensor(statistic) - output[tag][op] = statistic.astype(mstype.float32) + if isinstance(statistic, Tensor): + output[tag][op] = statistic.astype(mstype.float32) + else: + output[tag][op] = statistic def get_metrics(op_list, tag2tensor, eps, output=None): @@ -91,6 +94,9 @@ def validate_ops(ops): default_op = MonitorConst.OP_LIST[0] valid_ops.append(default_op) logger.info(f"There is no valid ops, default op {default_op} is used") + # 增加默认shape和dtype参数 + if "shape" not in valid_ops and "dtype" not in valid_ops: + valid_ops.extend(["shape", "dtype"]) return valid_ops @@ -171,7 +177,7 @@ def validate_alert(alert): args = rule.get("args") if args and isinstance(args, dict): threshold = args.get("threshold") - if not isinstance(threshold, float) or threshold < 0: + if not isinstance(threshold, (float, int)) or threshold < 0: raise TypeError('threshold must be float and not less than 0') dump = alert.get('dump') if dump and not isinstance(dump, bool): @@ -212,6 +218,11 @@ def validate_collect_times(collect_times): raise ValueError("collect_times must greater than 1") +def validate_dynamic_on(dynamic_on): + if not isinstance(dynamic_on, bool): + raise TypeError('dynamic_on should be a bool') + + def validate_config(config): config['ops'] = validate_ops(config.get('ops', [])) @@ -261,6 +272,9 @@ def validate_config(config): collect_times = config.get('collect_times', int(1e8)) validate_collect_times(collect_times) + dynamic_on = config.get('dynamic_on', False) + validate_dynamic_on(dynamic_on) + if not targets: if xy_distribution: config["all_xy"] = True diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index f20ed804c5bb8d8fbe4dba3e208060e8f52a3120..3207a593908a2c5b0c3823cd62f53f240990db65 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -29,6 +29,7 @@ class TensorConfig(BaseConfig): self.check_mode = None self.file_format = json_config.get("file_format") self.check_config() + self._check_summary_mode() self._check_config() def _check_config(self): @@ -42,12 +43,23 @@ class StatisticsConfig(BaseConfig): self.file_format = None self.check_mode = None self.check_config() - self._check_config() + self._check_summary_mode() - def _check_config(self): - single_opt = ["statistics", "md5"] + self.tensor_list = json_config.get("tensor_list", []) + self._check_str_list_config(self.tensor_list, "tensor_list") + self.stat_cal_mode = json_config.get("device", "host") + self.device_stat_precision_mode = json_config.get("precision", "high") + self._check_stat_params() + + def _check_stat_params(self): + if self.stat_cal_mode not in ["device", "host"]: + raise Exception("Config param [device] is invalid, expected from [\"device\", \"host\"]") + if self.device_stat_precision_mode not in ["high", "low"]: + raise Exception("Config param [precision] is invalid, expected from [\"high\", \"low\"]") + + def _check_summary_mode(self): muti_opt = ["md5", "max", "min", "mean", "l2norm"] - if isinstance(self.summary_mode, str) and self.summary_mode not in single_opt: + if isinstance(self.summary_mode, str) and self.summary_mode not in Const.SUMMARY_MODE: raise Exception("summary_mode is invalid") if isinstance(self.summary_mode, list) and not all(opt in muti_opt for opt in self.summary_mode): raise Exception("summary_mode is invalid") diff --git a/debug/accuracy_tools/msprobe/mindspore/overflow_check/overflow_check_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/overflow_check/overflow_check_tool_factory.py index a2d3e290bd6b16b3deeb7f22a5e7d327ebaa2bc4..1a31c6e658e08eb8edc13f405422fd368db83dc0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/overflow_check/overflow_check_tool_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/overflow_check/overflow_check_tool_factory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from msprobe.core.common.log import logger from msprobe.mindspore.common.const import Const from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.overflow_check.kernel_graph_overflow_check import KernelGraphOverflowCheck @@ -44,6 +45,7 @@ class OverflowCheckToolFactory: raise Exception("Valid level is needed.") tool = tool.get(config.execution_mode) if not tool: - raise Exception(f"Overflow check is not supported in {config.execution_mode} mode " - f"when level is {config.level}.") + logger.error(f"Overflow check is not supported in {config.execution_mode} mode " + f"when level is {config.level}.") + raise ValueError return tool(config) diff --git a/debug/accuracy_tools/msprobe/mindspore/runtime.py b/debug/accuracy_tools/msprobe/mindspore/runtime.py index 0191a484cbc096b2e211b22b5abce147eac23b97..9ea2e5d32f9db0fe4cc13a26eca52026dae9e599 100644 --- a/debug/accuracy_tools/msprobe/mindspore/runtime.py +++ b/debug/accuracy_tools/msprobe/mindspore/runtime.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from msprobe.mindspore.common.const import Const + + class Runtime: step_count: int = 0 rank_id: int = -1 is_running: bool = False + run_mode: str = Const.PYNATIVE_MODE diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 11d6db7a981a950120731df26edabee2e751b720..47d94a5be007c15288388eeeaa4b2ea45391652b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -32,20 +32,28 @@ else: from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException from msprobe.core.common.file_utils import create_directory -from msprobe.core.common.utils import Const, print_tools_ends_info, DumpPathAggregation +from msprobe.core.common.utils import Const, print_tools_ends_info, DumpPathAggregation, replace_last_occurrence from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.data_processor.base import (ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs, ModuleBackwardInputs) from msprobe.core.data_dump.scope import BaseScope +from msprobe.core.data_dump.api_registry import ApiRegistry from msprobe.mindspore.cell_processor import CellProcessor +from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.common.log import logger -from msprobe.mindspore.common.utils import (get_rank_if_initialized, clean_input_kwargs, - is_mindtorch, register_backward_hook_functions) -from msprobe.mindspore.dump.hook_cell.api_register import get_api_register +from msprobe.mindspore.common.utils import ( + get_rank_if_initialized, + clean_input_kwargs, + is_mindtorch, + get_cells_and_names_with_index, + has_kwargs_in_forward_hook +) +from msprobe.mindspore.dump.hook_cell.api_register import get_api_register, ApiTemplate from msprobe.mindspore.dump.hook_cell.primitive_hooks import PrimitiveHookService from msprobe.mindspore.dump.jit_dump import JitDump from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell from msprobe.mindspore.dump.kernel_dump.kernel_config import create_kernel_config_json +from msprobe.mindspore.runtime import Runtime if is_mindtorch(): import torch @@ -65,6 +73,7 @@ class Service: self.current_iter = 0 self.loop = 0 self.init_step = 0 + self.cur_token_id = 0 self.first_start = True self.current_rank = None self.dump_iter_dir = None @@ -75,10 +84,16 @@ class Service: # 提前注册，确保注册尽可能多的API hook self.api_register = get_api_register() self.register_api_hook() - self.init_for_debug_level() + self.currrent_step_first_debug_save = True + self.debug_variable_counter = None + self.ori_customer_func = {} @staticmethod - def check_model_valid(models): + def check_model_valid(models, token_range=None): + if token_range and not models: + error_info = "The 'model' parameter must be provided when token_range is not None" + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, error_info) + target_module_type = (torch.nn.Module, "torch.nn.Module") if is_mindtorch() else (nn.Cell, "mindspore.nn.Cell") if models is None or isinstance(models, target_module_type[0]): return models @@ -98,32 +113,21 @@ class Service: MsprobeException.INVALID_PARAM_ERROR, error_info) return models - @staticmethod - def prepare_module_input_output(target_type, cell, input_data, output): - if target_type == BaseScope.Module_Type_Module: - module_input_output = ModuleForwardInputsOutputs(args=input_data, kwargs={}, output=output) - else: - module_input_output = ModuleForwardInputsOutputs(args=input_data, kwargs=cell.input_kwargs, output=output) - return module_input_output - def build_hook(self, target_type, name): def pre_hook(api_or_cell_name, cell, input_data): - if not self.should_execute_hook(target_type, cell, True): - clean_input_kwargs(cell) - return None + if target_type == BaseScope.Module_Type_Module or \ + not self.should_execute_hook(target_type, cell, True): + return with _no_grad(): self.inner_switch = True - if target_type == BaseScope.Module_Type_Module: - api_or_cell_name = self.cell_processor.set_and_get_reserved_name(cell, api_or_cell_name) - else: - cell.forward_data_collected = True - HOOKCell.add_cell_count(name) - module_input_output = self.prepare_module_input_output(target_type, cell, input_data, None) + cell.forward_data_collected = True + HOOKCell.add_cell_count(name) + kwargs = cell.msprobe_input_kwargs if hasattr(cell, 'msprobe_input_kwargs') else {} + module_input_output = ModuleForwardInputsOutputs(args=input_data, kwargs=kwargs, output=None) self.data_collector.update_api_or_module_name(api_or_cell_name) self.data_collector.forward_input_data_collect(api_or_cell_name, cell, pid, module_input_output) self.inner_switch = False - return input_data def grad_hook(cell, ori_name, param_name): def hook_fn(grad): @@ -170,15 +174,20 @@ class Service: # 记录当前模块的参数梯度信息已占位 self.params_grad_info[grad_name] = True - def forward_hook(api_or_cell_name, cell, input_data, output): + def forward_hook(api_or_cell_name, cell, args, kwargs_or_output, output_or_kwargs): if not self.should_execute_hook(target_type, cell, True): clean_input_kwargs(cell) return None with _no_grad(): self.inner_switch = True - module_input_output = self.prepare_module_input_output(target_type, cell, input_data, output) + if not has_kwargs_in_forward_hook() or target_type == BaseScope.Module_Type_API: + kwargs = cell.msprobe_input_kwargs if hasattr(cell, 'msprobe_input_kwargs') else {} + output = kwargs_or_output + else: + kwargs = kwargs_or_output + output = output_or_kwargs + module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=output) if target_type == BaseScope.Module_Type_Module: - api_or_cell_name = self.cell_processor.set_and_get_reserved_name(cell, api_or_cell_name) params_dict = {} if self.config.task != Const.STRUCTURE: params_dict = { @@ -200,11 +209,13 @@ class Service: self.data_collector.update_api_or_module_name(api_or_cell_name) self.data_collector.forward_output_data_collect(api_or_cell_name, cell, pid, module_input_output) + clean_input_kwargs(cell) + if self.data_collector.if_return_forward_new_output(): forward_new_output = self.data_collector.get_forward_new_output() self.inner_switch = False return forward_new_output - clean_input_kwargs(cell) + self.inner_switch = False return output @@ -217,7 +228,6 @@ class Service: if target_type == BaseScope.Module_Type_Module: if not hasattr(cell, 'has_pre_hook_called') or not cell.has_pre_hook_called: need_exchange = False - api_or_cell_name = self.cell_processor.set_and_get_reserved_name(cell, api_or_cell_name) self.data_collector.update_api_or_module_name(api_or_cell_name) if self.data_collector: @@ -240,12 +250,11 @@ class Service: self.inner_switch = False pid = os.getpid() - if target_type == BaseScope.Module_Type_Module: - full_forward_name = name + Const.FORWARD - full_backward_name = name + Const.BACKWARD - else: + full_forward_name = name + if target_type == BaseScope.Module_Type_API: full_forward_name = name + str(HOOKCell.get_cell_count(name)) + Const.SEP + Const.FORWARD - full_backward_name = name + str(HOOKCell.get_cell_count(name)) + Const.SEP + Const.BACKWARD + full_backward_name = replace_last_occurrence(full_forward_name, Const.FORWARD, Const.BACKWARD) + pre_forward_hook = functools.partial(pre_hook, full_forward_name) forward_hook = functools.partial(forward_hook, full_forward_name) backward_hook = functools.partial(backward_hook, full_backward_name) @@ -254,8 +263,8 @@ class Service: def wrap_pre_forward_hook(cell, input_data): return pre_forward_hook(cell, input_data) - def wrap_forward_hook(cell, input_data, output_data): - return forward_hook(cell, input_data, output_data) + def wrap_forward_hook(cell, args, kwargs_or_output, output_or_kwargs=None): + return forward_hook(cell, args, kwargs_or_output, output_or_kwargs) def wrap_backward_hook(cell, grad_input, grad_output): return backward_hook(cell, grad_input, grad_output) @@ -272,17 +281,26 @@ class Service: self.primitive_counters[primitive_name] += 1 def step(self): - if self.config.level == Const.LEVEL_DEBUG: - return - if self.config.async_dump: - self.data_collector.fill_stack_tensor_data() - if self.config.task == Const.TENSOR: - self.data_collector.data_processor.dump_async_data() + if self.config.async_dump and self.config.task in [Const.STATISTICS, Const.TENSOR]: + self.data_collector.data_processor.dump_async_data() self.data_collector.write_json() + self.currrent_step_first_debug_save = True self.loop += 1 self.reset_status() - def start(self, model=None): + def start(self, model=None, token_range=None): + if self.current_iter == 0: + if not is_mindtorch() and self.config.level in [Const.LEVEL_MIX, Const.LEVEL_L1]: + JitDump.set_config(self.config) + JitDump.set_data_collector(self.data_collector) + if hasattr(ms.common.api, "_MindsporeFunctionExecutor"): + ms.common.api._MindsporeFunctionExecutor = JitDump + else: + ms.common.api._JitExecutor = JitDump + ms.common.api._PyNativeExecutor.grad = JitDump.grad + if pijit_label: + PIJitCaptureContext.__enter__ = self.empty + PIJitCaptureContext.__exit__ = self.empty self.current_iter = self.loop + self.init_step self.data_collector.update_iter(self.current_iter) if self.config.level == Const.LEVEL_DEBUG: @@ -297,11 +315,13 @@ class Service: print_tools_ends_info() return if self.config.step and self.current_iter not in self.config.step: + JitDump.jit_dump_switch = False return - self.model = self.check_model_valid(model) + self.model = self.check_model_valid(model, token_range) logger.info(f"{Const.TOOL_NAME}: debugger.start() is set successfully") + self.cur_token_id = 0 if self.first_start: try: self.current_rank = get_rank_if_initialized() @@ -310,28 +330,23 @@ class Service: if self.config.rank and self.current_rank not in self.config.rank: return + self.register_primitive_hook() - self.register_cell_hook() - if self.config.level in [Const.LEVEL_MIX, Const.LEVEL_L1]: - JitDump.set_config(self.config) - JitDump.set_data_collector(self.data_collector) - if hasattr(ms.common.api, "_MindsporeFunctionExecutor"): - ms.common.api._MindsporeFunctionExecutor = JitDump - else: - ms.common.api._JitExecutor = JitDump - ms.common.api._PyNativeExecutor.grad = JitDump.grad - if pijit_label: - PIJitCaptureContext.__enter__ = self.empty - PIJitCaptureContext.__exit__ = self.empty + if self.config.level in [Const.LEVEL_MIX, Const.LEVEL_L0]: + self.cell_processor.register_cell_hook(self.model, self.build_hook, self.config) self.first_start = False + if token_range: + self.register_infer_count_hook(self.model, token_range) + self.api_register.register_all_api() - self.switch = True - self.primitive_switch = True + if token_range is None: + self.switch = True + self.primitive_switch = True + JitDump.jit_dump_switch = True logger.info(f"Dump switch is turned on at step {self.current_iter}. ") self.create_dirs() logger.info(f"Dump data will be saved in {self.dump_iter_dir}.") - JitDump.jit_dump_switch = True def stop(self): if self.config.level == Const.LEVEL_DEBUG: @@ -350,10 +365,8 @@ class Service: self.switch = False self.primitive_switch = False self.start_call = False - if self.config.async_dump: - self.data_collector.fill_stack_tensor_data() - if self.config.task == Const.TENSOR: - self.data_collector.data_processor.dump_async_data() + if self.config.async_dump and self.config.task in [Const.STATISTICS, Const.TENSOR]: + self.data_collector.data_processor.dump_async_data() self.data_collector.write_json() JitDump.jit_dump_switch = False @@ -381,7 +394,11 @@ class Service: def create_dirs(self): create_directory(self.config.dump_path) - self.dump_iter_dir = os.path.join(self.config.dump_path, f"step{self.current_iter}") + if Runtime.run_mode == MsConst.PYNATIVE_GRAPH_MODE: + self.dump_iter_dir = os.path.join(self.config.dump_path, MsConst.PYNATIVE_MODE, f"step{self.current_iter}") + else: + self.dump_iter_dir = os.path.join(self.config.dump_path, f"step{self.current_iter}") + cur_rank = self.current_rank if self.current_rank is not None else '' if self.config.level == Const.LEVEL_L2: create_directory(self.dump_iter_dir) @@ -391,16 +408,20 @@ class Service: dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}") create_directory(dump_dir) - if self.config.task in self.data_collector.tasks_need_tensor_data: + + dump_data_dir = None + if self.config.task in self.data_collector.tasks_need_tensor_data or ( + self.config.task == Const.STATISTICS and self.config.tensor_list): dump_data_dir = os.path.join(dump_dir, "dump_tensor_data") create_directory(dump_data_dir) - else: - dump_data_dir = None dump_path_aggregation = DumpPathAggregation() - dump_path_aggregation.dump_file_path = os.path.join(dump_dir, "dump.json") - dump_path_aggregation.stack_file_path = os.path.join(dump_dir, "stack.json") - dump_path_aggregation.construct_file_path = os.path.join(dump_dir, "construct.json") + if self.config.level != Const.LEVEL_DEBUG: + dump_path_aggregation.dump_file_path = os.path.join(dump_dir, "dump.json") + dump_path_aggregation.stack_file_path = os.path.join(dump_dir, "stack.json") + dump_path_aggregation.construct_file_path = os.path.join(dump_dir, "construct.json") + else: + dump_path_aggregation.debug_file_path = os.path.join(dump_dir, "debug.json") dump_path_aggregation.dump_tensor_data_dir = dump_data_dir self.data_collector.update_dump_paths(dump_path_aggregation) @@ -417,19 +438,6 @@ class Service: self.api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) self.api_register.register_all_api() - def get_cells_and_names(self): - cells_and_names_with_index = {} - - def get_cell_or_module(model): - return model.named_modules() if is_mindtorch() else model.cells_and_names() - - if isinstance(self.model, (list, tuple)): - for index, model in enumerate(self.model): - cells_and_names_with_index[str(index)] = get_cell_or_module(model) - else: - cells_and_names_with_index["-1"] = get_cell_or_module(self.model) - return cells_and_names_with_index - def register_primitive_hook(self): if self.config.level not in [Const.LEVEL_MIX, Const.LEVEL_L1]: return @@ -437,7 +445,7 @@ class Service: return primitive_set = set() - cells_and_names_with_index = self.get_cells_and_names() + cells_and_names_with_index, _ = get_cells_and_names_with_index(self.model) for cells_and_names in cells_and_names_with_index.values(): for _, cell in cells_and_names: for attribute, value in vars(cell).items(): @@ -452,35 +460,31 @@ class Service: primitive_combined_name)}) primitive.__class__ = new_primitive - def register_cell_hook(self): - if self.config.level in [Const.LEVEL_MIX, Const.LEVEL_L0]: - logger.info(f"The cell {self.config.task} hook function is successfully mounted to the model.") - if not self.model: - raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, - f"The current level is {self.config.level}, the model cannot be None") - model_type = Const.MODULE if is_mindtorch() else Const.CELL - cells_and_names_with_index = self.get_cells_and_names() - - for index, cells_and_names in cells_and_names_with_index.items(): - model = self.model if index == "-1" else self.model[int(index)] - for name, cell in cells_and_names: - if cell == model: - continue - cell_index = (index + Const.SEP) if index != "-1" else "" - prefix = (model_type + Const.SEP + cell_index + name + - Const.SEP + cell.__class__.__name__ + Const.SEP) - _, forward_hook, backward_hook, _ = self.build_hook(BaseScope.Module_Type_Module, prefix) - cell.register_forward_hook(forward_hook) - cell.register_forward_pre_hook( - self.cell_processor.node_hook(prefix + Const.FORWARD, Const.START)) - cell.register_forward_hook( - self.cell_processor.node_hook(prefix + Const.FORWARD, Const.STOP)) - - register_backward_hook_functions["full"](cell, backward_hook) - register_backward_hook_functions["pre"]( - cell, self.cell_processor.node_hook(prefix + Const.BACKWARD, Const.START)) - register_backward_hook_functions["full"]( - cell, self.cell_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) + def register_infer_count_hook(self, root_model, token_range): + """ + 通过root_model执行的轮次来判断当前在第几个token + param root_model: 需要采集的推理模型 + param token_range: [start, end], 采集infer的token循环范围，左右皆包含在内 + return: None + """ + def infer_hook(model, args): + if self.cur_token_id == token_range[0]: + self.switch = True + self.primitive_switch = True + JitDump.jit_dump_switch = True + logger.info(f"Current token id: {self.cur_token_id}, start dump infer token.") + elif token_range[0] < self.cur_token_id <= token_range[1]: + logger.debug(f"Current token id: {self.cur_token_id}.") + elif self.cur_token_id == token_range[1] + 1: + self.switch = False + self.primitive_switch = False + JitDump.jit_dump_switch = False + logger.info(f"Current token id: {self.cur_token_id}, exceed token_range, early stop dump infer token.") + self.cur_token_id += 1 + if isinstance(root_model, list): + root_model = root_model[0] + logger.warning("Infer model can only input one to support token_range, choose the first one.") + root_model.register_forward_pre_hook(infer_hook) def reset_status(self): self.primitive_hook_service.primitive_counters.clear() @@ -495,33 +499,6 @@ class Service: if self.config.rank and self.current_rank not in self.config.rank: return - def init_for_debug_level(self): - if not (self.config.level == Const.LEVEL_DEBUG and self.config.task in [Const.TENSOR, Const.STATISTICS]): - return - try: - self.current_rank = get_rank_if_initialized() - except DistributedNotInitializedError: - self.current_rank = None - # dir: dump_path -- rank{} -- debug.json - self.dump_iter_dir = self.config.dump_path - cur_rank = self.current_rank if self.current_rank is not None else '' - dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}") - create_directory(dump_dir) - if self.config.task in self.data_collector.tasks_need_tensor_data: - dump_data_dir = os.path.join(dump_dir, "dump_tensor_data") - create_directory(dump_data_dir) - else: - dump_data_dir = None - - dump_path_aggregation = DumpPathAggregation() - dump_path_aggregation.dump_tensor_data_dir = dump_data_dir - dump_path_aggregation.debug_file_path = os.path.join(dump_dir, "debug.json") - self.data_collector.update_dump_paths(dump_path_aggregation) - self.data_collector.initialize_json_file( - framework=Const.MT_FRAMEWORK if is_mindtorch() else Const.MS_FRAMEWORK - ) - self.debug_variable_counter = defaultdict(int) - def save(self, variable, name, save_backward): ''' Args: @@ -533,6 +510,21 @@ class Service: ''' if self.config.level != Const.LEVEL_DEBUG: return + + self.current_iter = self.loop + self.init_step + if self.config.step and self.current_iter not in self.config.step: + return + + if self.currrent_step_first_debug_save: + try: + self.current_rank = get_rank_if_initialized() + except DistributedNotInitializedError: + self.current_rank = None + + self.create_dirs() + self.debug_variable_counter = defaultdict(int) + self.currrent_step_first_debug_save = False + count = self.debug_variable_counter[name] self.debug_variable_counter[name] += 1 @@ -545,3 +537,13 @@ class Service: # backward save if save_backward: self.data_collector.debug_data_collect_backward(variable, grad_name_with_count) + + def register_custom_api(self, module, api_name, api_prefix): + self.ori_customer_func[str(module) + Const.SEP + api_name] = getattr(module, api_name) + ApiRegistry.register_custom_api(module, api_name, api_prefix, + functools.partial(self.build_hook, BaseScope.Module_Type_API), ApiTemplate) + + def restore_custom_api(self, module, api): + ori_func = self.ori_customer_func.get(str(module) + Const.SEP + api) + if ori_func: + setattr(module, api, ori_func) diff --git a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py index a9cb5e6dd4037dcdeffe3c4d9584ad93c42022d6..10b74ea22b02d0668d0b3b17a569c5e1a67c1dd8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py @@ -29,11 +29,14 @@ class TaskHandlerFactory: } @staticmethod - def create(config: DebuggerConfig): + def create(config: DebuggerConfig, model=None): task = TaskHandlerFactory.tasks.get(config.task) if not task: raise Exception("Valid task is needed.") - handler = task.create(config) + if task == DumpToolFactory: + handler = task.create(config, model) + else: + handler = task.create(config) if not handler: raise Exception("Can not find task handler") return handler diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 8e0386fde6dccc071c3d9d8e1a86729a2c483c7c..1af4e00c926352f6d2d0348c8a744a8f6f93fdd1 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -22,6 +22,8 @@ from msprobe.core.common.log import logger from msprobe.core.compare.utils import _compare_parser from msprobe.core.compare.compare_cli import compare_cli from msprobe.core.compare.merge_result.merge_result_cli import _merge_result_parser, merge_result_cli +from msprobe.core.config_check.config_check_cli import _config_checking_parser, \ + _run_config_checking_command def is_module_available(module_name): @@ -51,6 +53,8 @@ def main(): graph_service_cmd_parser = subparsers.add_parser('graph') op_generate_cmd_parser = subparsers.add_parser('op_generate') merge_result_parser = subparsers.add_parser('merge_result') + config_checking_parser = subparsers.add_parser('config_check') + _config_checking_parser(config_checking_parser) _compare_parser(compare_cmd_parser) _merge_result_parser(merge_result_parser) @@ -91,6 +95,10 @@ def main(): _ms_graph_service_parser(graph_service_cmd_parser) + from msprobe.mindspore.api_accuracy_checker.generate_op_script.op_generator import _op_generator_parser, \ + _run_operator_generate_commond + _op_generator_parser(op_generate_cmd_parser) + args = parser.parse_args(sys.argv[1:]) if sys.argv[2] == Const.PT_FRAMEWORK: if not is_torch_available: @@ -118,6 +126,8 @@ def main(): compare_cli(args) elif sys.argv[3] == "merge_result": merge_result_cli(args) + elif sys.argv[3] == "config_check": + _run_config_checking_command(args) else: if not is_module_available(Const.MS_FRAMEWORK): logger.error("MindSpore does not exist, please install MindSpore library") @@ -134,9 +144,13 @@ def main(): mul_api_checker_main(args) elif sys.argv[3] == "graph": _ms_graph_service_command(args) + elif sys.argv[3] == 'op_generate': + _run_operator_generate_commond(args) elif sys.argv[3] == "code_mapping": from msprobe.mindspore.code_mapping.main import code_mapping_main code_mapping_main(args) + elif sys.argv[3] == "config_check": + _run_config_checking_command(args) if __name__ == "__main__": diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index f2b2d6a30463c62846bcc02e147c9c319f55d1b8..1e844ff81a8543c9865dbefc3c39c12202d2c6e2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -125,8 +125,8 @@ class CheckerConfig: save_error_data=config_params.get('save_error_data'), is_continue_run_ut=config_params.get('is_continue_run_ut'), real_data_path=config_params.get('real_data_path'), - white_list=self.white_list, - black_list=self.black_list, + white_list=self.white_list.copy() if self.white_list else [], + black_list=self.black_list.copy() if self.black_list else [], error_data_path=config_params.get('error_data_path'), online_config=self.get_online_config() ) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index cd60d8bc15f5b1c2889c8bdf96d3e9490ce09498..55e93d271cec67334fe21c1f6466df2d0254a36b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -40,7 +40,7 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validat from msprobe.pytorch.api_accuracy_checker.common.utils import extract_detailed_api_segments, extract_basic_api_segments from msprobe.core.common.file_utils import FileChecker, change_mode, create_directory from msprobe.pytorch.common.log import logger -from msprobe.core.common.utils import CompareException +from msprobe.core.common.utils import CompareException, check_op_str_pattern_valid from msprobe.core.common.const import Const, CompareConst, FileCheckConst CompareConfig = namedtuple('CompareConfig', ['npu_csv_path', 'gpu_csv_path', 'result_csv_path', 'details_csv_path']) @@ -151,6 +151,7 @@ def analyse_csv(npu_data, gpu_data, config): message = '' compare_column = ApiPrecisionOutputColumn() full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] + check_op_str_pattern_valid(full_api_name_with_direction_status) row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status] api_name, api_full_name, direction_status = extract_detailed_api_segments(full_api_name_with_direction_status) if not api_full_name: diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index cf5928e509e3138ea762cd9d7af6fc26a5d2c5c9..c12a54c18ad07ae302b41d12704dc82fec01b4c2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -40,6 +40,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dty DETAIL_TEST_ROWS, BENCHMARK_COMPARE_SUPPORT_LIST from msprobe.pytorch.api_accuracy_checker.common.utils import extract_basic_api_segments from msprobe.pytorch.common.log import logger +from msprobe.core.common.decorator import recursion_depth_decorator ResultInfo = namedtuple('ResultInfo', ['full_api_name', 'fwd_success_status', 'bwd_success_status', @@ -178,6 +179,41 @@ class Comparator: if not os.path.exists(detail_save_path): write_csv(DETAIL_TEST_ROWS, detail_save_path) + @recursion_depth_decorator("compare_core") + def _compare_core(self, api_name, bench_output, device_output): + compare_column = CompareColumn() + if not isinstance(bench_output, type(device_output)): + status = CompareConst.ERROR + message = "bench and npu output type is different." + elif isinstance(bench_output, dict): + b_keys, n_keys = set(bench_output.keys()), set(device_output.keys()) + if b_keys != n_keys: + status = CompareConst.ERROR + message = "bench and npu output dict keys are different." + else: + status, compare_column, message = self._compare_core(api_name, list(bench_output.values()), + list(device_output.values())) + elif isinstance(bench_output, torch.Tensor): + copy_bench_out = bench_output.detach().clone() + copy_device_output = device_output.detach().clone() + compare_column.bench_type = str(copy_bench_out.dtype) + compare_column.npu_type = str(copy_device_output.dtype) + compare_column.shape = tuple(device_output.shape) + status, compare_column, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output, + compare_column) + elif isinstance(bench_output, (bool, int, float, str)): + compare_column.bench_type = str(type(bench_output)) + compare_column.npu_type = str(type(device_output)) + status, compare_column, message = self._compare_builtin_type(bench_output, device_output, compare_column) + elif bench_output is None: + status = CompareConst.SKIP + message = "Bench output is None, skip this test." + else: + status = CompareConst.ERROR + message = "Unexpected output type in compare_core: {}".format(type(bench_output)) + + return status, compare_column, message + def write_summary_csv(self, test_result): test_rows = [] try: @@ -293,40 +329,6 @@ class Comparator: test_final_success = CompareConst.WARNING return test_final_success, detailed_result_total - def _compare_core(self, api_name, bench_output, device_output): - compare_column = CompareColumn() - if not isinstance(bench_output, type(device_output)): - status = CompareConst.ERROR - message = "bench and npu output type is different." - elif isinstance(bench_output, dict): - b_keys, n_keys = set(bench_output.keys()), set(device_output.keys()) - if b_keys != n_keys: - status = CompareConst.ERROR - message = "bench and npu output dict keys are different." - else: - status, compare_column, message = self._compare_core(api_name, list(bench_output.values()), - list(device_output.values())) - elif isinstance(bench_output, torch.Tensor): - copy_bench_out = bench_output.detach().clone() - copy_device_output = device_output.detach().clone() - compare_column.bench_type = str(copy_bench_out.dtype) - compare_column.npu_type = str(copy_device_output.dtype) - compare_column.shape = tuple(device_output.shape) - status, compare_column, message = self._compare_torch_tensor(api_name, copy_bench_out, copy_device_output, - compare_column) - elif isinstance(bench_output, (bool, int, float, str)): - compare_column.bench_type = str(type(bench_output)) - compare_column.npu_type = str(type(device_output)) - status, compare_column, message = self._compare_builtin_type(bench_output, device_output, compare_column) - elif bench_output is None: - status = CompareConst.SKIP - message = "Bench output is None, skip this test." - else: - status = CompareConst.ERROR - message = "Unexpected output type in compare_core: {}".format(type(bench_output)) - - return status, compare_column, message - def _compare_torch_tensor(self, api_name, bench_output, device_output, compare_column): cpu_shape = bench_output.shape npu_shape = device_output.shape diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py index 641eada030353ec67f6ce7b59bd3d14909a56e51..c58c058674f31d8acb24a008104cdd32b1969726 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/generate_op_script/op_generator.py @@ -28,10 +28,10 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import binary_st ulp_standard_api, thousandth_standard_api from msprobe.core.common.file_utils import FileOpen, load_json, save_json from msprobe.core.common.utils import check_file_or_directory_path, check_op_str_pattern_valid, is_int -from msprobe.core.common.const import Const, MonitorConst, MsgConst +from msprobe.core.common.const import Const, MonitorConst, MsgConst, FileCheckConst from msprobe.core.common.log import logger -from msprobe.core.common.file_utils import make_dir -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.file_utils import make_dir, change_mode +from msprobe.core.common.decorator import recursion_depth_decorator TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"] TORCH_BOOL_TYPE = ["torch.bool"] @@ -50,6 +50,7 @@ DATA_NAME = "data_name" API_MAX_LENGTH = 30 PROPAGATION_LIST = [Const.FORWARD, Const.BACKWARD] DATAMODE_LIST = ["random_data", "real_data"] +ITER_MAX_TIMES = 1000 class APIInfo: @@ -97,6 +98,8 @@ class CommonConfig: iter_t = self.iter_times if iter_t <= 0: raise ValueError("iter_times should be an integer bigger than zero!") + if iter_t > ITER_MAX_TIMES: + raise ValueError("iter_times should not be greater than 1000!") json_file = self.extract_api_path propagation = self.propagation @@ -117,7 +120,7 @@ class CommonConfig: # Retrieve the first API name and dictionary forward_item = next(iter(json_content.items()), None) - if not forward_item or not isinstance(forward_item[1], dict): + if not forward_item or not isinstance(forward_item[1], dict) or not forward_item[1]: raise ValueError(f'Invalid forward API data in json_content!') # if propagation is backward, ensure json file contains forward and backward info @@ -127,7 +130,7 @@ class CommonConfig: # if propagation is backward, ensure it has valid data if propagation == Const.BACKWARD: backward_item = list(json_content.items())[1] - if not isinstance(backward_item[1], dict): + if not isinstance(backward_item[1], dict) or not backward_item[1]: raise ValueError(f'Invalid backward API data in json_content!') return json_content @@ -169,7 +172,7 @@ class APIExtractor: value = self.load_real_data_path(value, real_data_path) new_data[key] = value if not new_data: - logger.error(f"Error: The api '{self.api_name}' does not exist in the file.") + logger.warning(f"Warning: The api '{self.api_name}' does not exist in the file.") else: save_json(self.output_file, new_data, indent=4) logger.info( @@ -408,19 +411,16 @@ class OperatorScriptGenerator: return kwargs_dict_generator - def _op_generator_parser(parser): - parser.add_argument("-i", "--config_input", dest="config_input", default='', type=str, - help=" Path of config json file", required=True) + parser.add_argument("-i", "--config_input", dest="config_input", type=str, + help=" Path of config json file", required=True) parser.add_argument("-o", "--api_output_path", dest="api_output_path", type=str, - help=" Path of extract api_name.json.", - required=True) + help=" Path of extract api_name.json.", required=True) def parse_json_config(json_file_path): if not json_file_path: - config_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - json_file_path = os.path.join(config_dir, "config.json") + raise Exception("config_input path can not be empty, please check.") json_config = load_json(json_file_path) common_config = CommonConfig(json_config) return common_config @@ -468,6 +468,7 @@ def _run_operator_generate_commond(cmd_args): fout.write(code_template.format(**internal_settings)) except OSError: logger.error(f"Failed to open file. Please check file {template_path} or {operator_script_path}.") + change_mode(operator_script_path, FileCheckConst.DATA_FILE_AUTHORITY) logger.info(f"Generate operator script successfully and the name is {operator_script_path}.") diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template index 131fd211ad82dad8256c48e59195fc335efa936b..c60d84994745e94bef6d05a78d83fae81df7ed1e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/generate_op_script/operator_replication.template @@ -1,6 +1,6 @@ -import json import os -import math +import re +import stat from enum import Enum, auto import torch try: @@ -25,6 +25,31 @@ RAISE_PRECISION = {{ }} THOUSANDTH_THRESHOLDING = 0.001 BACKWARD = 'backward' +DIR = "dir" +FILE = "file" +READ_ABLE = "read" +WRITE_ABLE = "write" +READ_WRITE_ABLE = "read and write" +DIRECTORY_LENGTH = 4096 +FILE_NAME_LENGTH = 255 +SOFT_LINK_ERROR = "检测到软链接" +FILE_PERMISSION_ERROR = "文件权限错误" +INVALID_FILE_ERROR = "无效文件" +ILLEGAL_PATH_ERROR = "非法文件路径" +ILLEGAL_PARAM_ERROR = "非法打开方式" +FILE_TOO_LARGE_ERROR = "文件过大" +FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" +FILE_SIZE_DICT = {{ + ".pkl": 1073741824, # 1 * 1024 * 1024 * 1024 + ".npy": 10737418240, # 10 * 1024 * 1024 * 1024 + ".json": 1073741824, # 1 * 1024 * 1024 * 1024 + ".pt": 10737418240, # 10 * 1024 * 1024 * 1024 + ".csv": 1073741824, # 1 * 1024 * 1024 * 1024 + ".xlsx": 1073741824, # 1 * 1024 * 1024 * 1024 + ".yaml": 1073741824, # 1 * 1024 * 1024 * 1024 + ".ir": 1073741824 # 1 * 1024 * 1024 * 1024 +}} +COMMOM_FILE_SIZE = 1048576 # 1 * 1024 * 1024 class CompareStandard(Enum): BINARY_EQUALITY_STANDARD = auto() @@ -33,13 +58,189 @@ class CompareStandard(Enum): BENCHMARK_STANDARD = auto() THOUSANDTH_STANDARD = auto() +class FileChecker: + """ + The class for check file. + + Attributes: + file_path: The file or dictionary path to be verified. + path_type: file or dictionary + ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability + file_type(str): The correct file type for file + """ + + def __init__(self, file_path, path_type, ability=None, file_type=None, is_script=True): + self.file_path = file_path + self.path_type = self._check_path_type(path_type) + self.ability = ability + self.file_type = file_type + self.is_script = is_script + + @staticmethod + def _check_path_type(path_type): + if path_type not in [DIR, FILE]: + print(f'ERROR: The path_type must be {{DIR}} or {{FILE}}.') + raise Exception(ILLEGAL_PARAM_ERROR) + return path_type + + def common_check(self): + """ + 功能：用户校验基本文件权限：软连接、文件长度、是否存在、读写权限、文件属组、文件特殊字符 + 注意：文件后缀的合法性，非通用操作，可使用其他独立接口实现 + """ + FileChecker.check_path_exists(self.file_path) + FileChecker.check_link(self.file_path) + self.file_path = os.path.realpath(self.file_path) + FileChecker.check_path_length(self.file_path) + FileChecker.check_path_type(self.file_path, self.path_type) + self.check_path_ability() + if self.is_script: + FileChecker.check_path_owner_consistent(self.file_path) + FileChecker.check_path_pattern_valid(self.file_path) + FileChecker.check_common_file_size(self.file_path) + FileChecker.check_file_suffix(self.file_path, self.file_type) + if self.path_type == FILE: + FileChecker.check_dirpath_before_read(self.file_path) + return self.file_path + + def check_path_ability(self): + if self.ability == WRITE_ABLE: + FileChecker.check_path_writability(self.file_path) + if self.ability == READ_ABLE: + FileChecker.check_path_readability(self.file_path) + if self.ability == READ_WRITE_ABLE: + FileChecker.check_path_readability(self.file_path) + FileChecker.check_path_writability(self.file_path) + + @staticmethod + def check_path_exists(path): + if not os.path.exists(path): + print(f'ERROR: The file path %s does not exist.' % path) + raise Exception() + + @staticmethod + def check_link(path): + abs_path = os.path.abspath(path) + if os.path.islink(abs_path): + print('ERROR: The file path {{}} is a soft link.'.format(path)) + raise Exception(SOFT_LINK_ERROR) + + @staticmethod + def check_path_length(path, name_length=None): + file_max_name_length = name_length if name_length else FILE_NAME_LENGTH + if len(path) > DIRECTORY_LENGTH or \ + len(os.path.basename(path)) > file_max_name_length: + print(f'ERROR: The file path length exceeds limit.') + raise Exception(ILLEGAL_PATH_ERROR) + + @staticmethod + def check_path_type(file_path, file_type): + if file_type == FILE: + if not os.path.isfile(file_path): + print(f"ERROR: The {{file_path}} should be a file!") + raise Exception(INVALID_FILE_ERROR) + if file_type == DIR: + if not os.path.isdir(file_path): + print(f"ERROR: The {{file_path}} should be a dictionary!") + raise Exception(INVALID_FILE_ERROR) + + @staticmethod + def check_path_owner_consistent(path): + file_owner = os.stat(path).st_uid + if file_owner != os.getuid() and os.getuid() != 0: + print('ERROR: The file path %s may be insecure because is does not belong to you.' % path) + raise Exception(FILE_PERMISSION_ERROR) + + @staticmethod + def check_path_pattern_valid(path): + if not re.match(FILE_VALID_PATTERN, path): + print('ERROR: The file path %s contains special characters.' % (path)) + raise Exception(ILLEGAL_PATH_ERROR) + + @staticmethod + def check_common_file_size(file_path): + if os.path.isfile(file_path): + for suffix, max_size in FILE_SIZE_DICT.items(): + if file_path.endswith(suffix): + FileChecker.check_file_size(file_path, max_size) + return + FileChecker.check_file_size(file_path, COMMOM_FILE_SIZE) + + @staticmethod + def check_file_size(file_path, max_size): + try: + file_size = os.path.getsize(file_path) + except OSError as os_error: + print(f'ERROR: Failed to open "{{file_path}}". {{str(os_error)}}') + raise Exception(INVALID_FILE_ERROR) from os_error + if file_size >= max_size: + print(f'ERROR: The size ({{file_size}}) of {{file_path}} exceeds ({{max_size}}) bytes, tools not support.') + raise Exception(FILE_TOO_LARGE_ERROR) + + @staticmethod + def check_file_suffix(file_path, file_suffix): + if file_suffix: + if not file_path.endswith(file_suffix): + print(f"The {{file_path}} should be a {{file_suffix}} file!") + raise Exception(INVALID_FILE_ERROR) + + @staticmethod + def check_dirpath_before_read(path): + path = os.path.realpath(path) + dirpath = os.path.dirname(path) + if FileChecker.check_others_writable(dirpath): + print(f"WARNING: The directory is writable by others: {{dirpath}}.") + try: + FileChecker.check_path_owner_consistent(dirpath) + except Exception: + print(f"WARNING: The directory {{dirpath}} is not yours.") + + @staticmethod + def check_others_writable(directory): + dir_stat = os.stat(directory) + is_writable = ( + bool(dir_stat.st_mode & stat.S_IWGRP) or # 组可写 + bool(dir_stat.st_mode & stat.S_IWOTH) # 其他用户可写 + ) + return is_writable + + @staticmethod + def check_path_readability(path): + if not os.access(path, os.R_OK): + print('ERROR: The file path %s is not readable.' % path) + raise Exception(FILE_PERMISSION_ERROR) + + @staticmethod + def check_path_writability(path): + if not os.access(path, os.W_OK): + print('ERROR: The file path %s is not writable.' % path) + raise Exception(FILE_PERMISSION_ERROR) + + +def check_file_or_directory_path(path, isdir=False): + """ + Function Description: + check whether the path is valid + Parameter: + path: the path to check + isdir: the path is dir or file + Exception Description: + when invalid data throw exception + """ + if isdir: + path_checker = FileChecker(path, DIR, WRITE_ABLE) + else: + path_checker = FileChecker(path, FILE, READ_ABLE) + path_checker.common_check() + def load_pt(pt_path, to_cpu=False): pt_path = os.path.realpath(pt_path) + check_file_or_directory_path(pt_path) try: if to_cpu: - pt = torch.load(pt_path, map_location=torch.device("cpu")) + pt = torch.load(pt_path, map_location=torch.device("cpu"), weights_only=True) else: - pt = torch.load(pt_path) + pt = torch.load(pt_path, weights_only=True) except Exception as e: raise RuntimeError(f"load pt file {{pt_path}} failed") from e return pt @@ -202,6 +403,7 @@ def compare_tensor(out_device, out_bench, api_name): else: abs_err = torch.abs(out_device - out_bench) abs_bench = torch.abs(out_bench) + eps = 2 ** -23 if dtype_bench == torch.float32: eps = 2 ** -23 if dtype_bench == torch.float64: diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index 1354e2dea17439d89a938ab660b60c9b514d31dc..204ef90122b2b062e6b283b54084e4fb58449ea5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -87,10 +87,6 @@ def signal_handler(signum, frame): raise KeyboardInterrupt() -signal.signal(signal.SIGINT, signal_handler) -signal.signal(signal.SIGTERM, signal_handler) - - ParallelUTConfig = namedtuple('ParallelUTConfig', ['api_files', 'out_path', 'num_splits', 'save_error_data_flag', 'jit_compile_flag', 'device_id', 'result_csv_path', 'total_items', 'config_path']) @@ -100,7 +96,7 @@ def run_parallel_ut(config): processes = [] device_id_cycle = cycle(config.device_id) if config.save_error_data_flag: - logger.info("UT task error datas will be saved") + logger.info("UT task error data will be saved") logger.info(f"Starting parallel UT with {config.num_splits} processes") progress_bar = tqdm(total=config.total_items, desc="Total items", unit="items") @@ -132,6 +128,9 @@ def run_parallel_ut(config): sys.stdout.flush() except ValueError as e: logger.warning(f"An error occurred while reading subprocess output: {e}") + finally: + if process.poll() is None: + process.stdout.close() def update_progress_bar(progress_bar, result_csv_path): while any(process.poll() is None for process in processes): @@ -217,6 +216,8 @@ def prepare_config(args): def main(): + signal.signal(signal.SIGINT, signal_handler) + signal.signal(signal.SIGTERM, signal_handler) parser = argparse.ArgumentParser(description='Run UT in parallel') _run_ut_parser(parser) parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index f0490ed62edbaef51d1be10fdb7010a56d174041..0f184d14b66d84607a6767ba9ef5210ff4fc5b69 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -34,8 +34,10 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import exec_api, i from msprobe.core.common.file_utils import check_link, FileChecker from msprobe.pytorch.api_accuracy_checker.common.utils import extract_basic_api_segments from msprobe.core.common.const import FileCheckConst, Const +from msprobe.core.common.utils import check_op_str_pattern_valid from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward +from msprobe.core.common.decorator import recursion_depth_decorator def check_tensor_overflow(x): @@ -63,6 +65,7 @@ def check_tensor_overflow(x): return False +@recursion_depth_decorator("check_data_overflow") def check_data_overflow(x, device): if isinstance(x, (tuple, list)): if not x: @@ -75,6 +78,7 @@ def check_data_overflow(x, device): return torch_npu.npu.utils.npu_check_overflow(x) +@recursion_depth_decorator("is_bool_output") def is_bool_output(x): if isinstance(x, (tuple, list)): if not x: @@ -91,6 +95,7 @@ def run_overflow_check(forward_file): dump_path = os.path.dirname(forward_file) real_data_path = os.path.join(dump_path, Const.DUMP_TENSOR_DATA) for api_full_name, api_info_dict in tqdm(forward_content.items()): + check_op_str_pattern_valid(api_full_name) if is_unsupported_api(api_full_name, is_overflow_check=True): continue try: diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 36acb061025c21b2b87490be9f8ce8e29b6bcc80..52486480dcaf93d743fef2bb4de8a9a30a7ec90e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -49,7 +49,7 @@ from msprobe.core.common.file_utils import FileChecker, change_mode, \ from msprobe.pytorch.common.log import logger from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.const import Const, FileCheckConst, CompareConst -from msprobe.core.common.utils import safe_get_value, CompareException +from msprobe.core.common.utils import safe_get_value, CompareException, is_int, check_op_str_pattern_valid from msprobe.pytorch.common.utils import seed_all from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTL, ATTLConfig, move2device_exec from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.device_dispatch import ConsumerDispatcher @@ -65,7 +65,8 @@ DETAILS_FILE_NAME = "accuracy_checking_details_" + current_time + ".csv" not_backward_list = ['repeat_interleave'] unsupported_backward_list = ['masked_select'] -unsupported_api_list = ["to"] +unsupported_api_list = ["to", "empty", "empty_like", "empty_strided", "new_empty", "new_empty_strided", + "empty_with_format"] tqdm_params = { @@ -97,7 +98,7 @@ def run_ut(config): logger.info(f"UT task details will be saved in {config.details_csv_path}") if config.save_error_data: - logger.info(f"UT task error_datas will be saved in {config.error_data_path}") + logger.info(f"UT task error_data will be saved in {config.error_data_path}") compare = Comparator(config.result_csv_path, config.details_csv_path, config.is_continue_run_ut, config=config) if config.online_config.is_online: @@ -121,6 +122,7 @@ def run_ut(config): def run_api_offline(config, compare, api_name_set): err_column = CompareColumn() for _, (api_full_name, api_info_dict) in enumerate(tqdm(config.forward_content.items(), **tqdm_params)): + check_op_str_pattern_valid(api_full_name) if api_full_name in api_name_set: continue if is_unsupported_api(api_full_name): @@ -350,6 +352,9 @@ def need_to_backward(grad_index, out): def run_backward(args, grad, grad_index, out): if grad_index is not None: + if not is_int(grad_index): + logger.error(f"{grad_index} dtype is not int") + raise TypeError(f"{grad_index} dtype is not int") if grad_index >= len(out): logger.error(f"Run backward error when grad_index is {grad_index}") raise IndexError(f"Run backward error when grad_index is {grad_index}") @@ -436,6 +441,7 @@ def preprocess_forward_content(forward_content): arg_cache = {} for key, value in forward_content.items(): + check_op_str_pattern_valid(key) base_key = key.rsplit(Const.SEP, 1)[0] if key not in arg_cache: @@ -499,6 +505,7 @@ def checked_online_config(online_config): check_file_or_directory_path(os.path.join(online_config.tls_path, "server.key")) check_file_or_directory_path(os.path.join(online_config.tls_path, "server.crt")) check_crt_valid(os.path.join(online_config.tls_path, "server.crt")) + check_crt_valid(os.path.join(online_config.tls_path, "server.key"), True) # host and port if not isinstance(online_config.host, str) or not re.match(Const.ipv4_pattern, online_config.host): @@ -568,7 +575,15 @@ def run_ut_command(args): error_data_path = checker_config.error_data_path if save_error_data: if args.result_csv_path: - time_info = result_csv_path.split('.')[0].split('_')[-1] + parts_by_dot = result_csv_path.split(Const.SEP) + if len(parts_by_dot) < 2 or not parts_by_dot[0]: + raise ValueError("result_csv_path does not contain a valid file name with an extension.") + file_name_part = parts_by_dot[0] + parts_by_underscore = file_name_part.split(Const.REPLACEMENT_CHARACTER) + if len(parts_by_underscore) < 2: + raise ValueError("File name part does not contain enough '_' separated segments.") + time_info = parts_by_underscore[-1] + global UT_ERROR_DATA_DIR UT_ERROR_DATA_DIR = 'ut_error_data' + time_info error_data_path = initialize_save_error_data(error_data_path) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index e62315a1a163a1dbba605de31d3b01703314ee5b..289773d0e603192ffe5bf83447d7452b5fad4b37 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -124,8 +124,6 @@ def exec_api(exec_params): api_register.initialize_hook(None) api_func_type = list(prefix_map.keys())[list(prefix_map.values()).index(api_type)] api_func = api_register.ori_api_attr.get(Const.PT_FRAMEWORK + Const.SEP + api_func_type, {}).get(api_name) - if api_func is None: - return out torch_api = ApiTemplate(api_name, api_func, api_type, None, need_hook=False, device=device) if is_autocast: diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index f858067b6616ff34ee95e1c4394e63fe4385b397..2cfc355ec035d245261ca9c817e02687c684d471 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -27,8 +27,7 @@ from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.client import T from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server import TCPServer from msprobe.core.common.file_utils import remove_path from msprobe.pytorch.common.utils import logger, save_api_data, load_api_data, save_pkl, load_pkl -from msprobe.core.common.utils import recursion_depth_decorator - +from msprobe.core.common.decorator import recursion_depth_decorator BufferType = Union[ApiData, Dict[str, Any], str] # Union[Tensor, Tuple[Optional[Tensor]]] diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py index fbb087deec73bb6e77c0d7581128c74e2d9be9fa..8e173de2f219dfbb0ac23e1fa68c797a98494131 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib +import zlib import io import struct import time import os -import signal from queue import Queue from threading import Thread from typing import Union @@ -104,10 +103,11 @@ class TCPClient: self.factory = MessageClientFactory() self.factory.protocol = cur_protocol if self.tls_path: + from OpenSSL import SSL from twisted.internet import ssl - client_key = os.path.join(self.tls_path, "client.key") - client_crt = os.path.join(self.tls_path, "client.crt") client_context_factory = ssl.DefaultOpenSSLContextFactory(client_key, client_crt) + client_context_ = client_context_factory.getContext() + client_context_.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT) endpoint = endpoints.SSL4ClientEndpoint(reactor, self.host, self.port, client_context_factory) else: endpoint = endpoints.TCP4ClientEndpoint(reactor, self.host, self.port) @@ -299,12 +299,12 @@ class ClientProtocol(protocol.Protocol): def send_wrapped_data(self, data, sequence_number: int = 0, rank: int = 0, step: int = 0): length = len(data) - md5_hash = hashlib.md5(data).hexdigest() if self.check_sum else "" + data_crc = f"{zlib.crc32(data):08x}" if self.check_sum else "" data_meaasge = length.to_bytes(8, byteorder=bytes_order) + \ sequence_number.to_bytes(8, byteorder=bytes_order) + \ rank.to_bytes(8, byteorder=bytes_order) + \ step.to_bytes(8, byteorder=bytes_order) + \ - md5_hash.encode() + \ + data_crc.encode() + \ data logger.debug(f"send 流水号: {sequence_number}; RANK: {rank}; STEP: {step}; LENGTH: {length}") diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py index 411e36d4cb3014b75a46d58ebec99b7e8b7c7c44..d673d731157ce25d8b9b611f9138a8d8314f8180 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os.path import struct -import hashlib +import zlib import time import io from threading import Thread @@ -46,12 +45,11 @@ class TCPServer: if self.tls_path: from OpenSSL import SSL from twisted.internet import ssl - server_key = os.path.join(self.tls_path, "server.key") - server_crt = os.path.join(self.tls_path, "server.crt") server_context_factory = ssl.DefaultOpenSSLContextFactory(server_key, server_crt, SSL.TLSv1_2_METHOD) server_context_ = server_context_factory.getContext() server_context_.set_cipher_list(cipher_list) server_context_.set_options(SSL.OP_NO_RENEGOTIATION) + server_context_.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT) endpoint = endpoints.SSL4ServerEndpoint(reactor, self.port, server_context_factory) else: endpoint = endpoints.TCP4ServerEndpoint(reactor, self.port) @@ -85,10 +83,10 @@ class ServerProtocol(protocol.Protocol): self.consumer_queue = shared_queue self.check_sum = check_sum self.length_width = 8 - self.md5_width = 32 + self.crc_width = 8 self.obj_length = None self.tell = 0 - self.obj_md5 = None + self.obj_crc = None self.obj_body = None self.sequence_number = -1 self.rank = -1 @@ -99,7 +97,7 @@ class ServerProtocol(protocol.Protocol): self.buffer = io.BytesIO() self.obj_length = None self.tell = 0 - self.obj_md5 = None + self.obj_crc = None self.obj_body = None self.factory.transport_dict[self.transport] = 1 self.factory.transport_list.append(self.transport) @@ -132,11 +130,12 @@ class ServerProtocol(protocol.Protocol): time.sleep(0.1) obj_key = str(self.sequence_number) + "_" + str(self.rank) + "_" + str(self.step) + # get the crc value of a 16-bit string with a length of 8 + recv_crc = f"{zlib.crc32(self.obj_body):08x}" - recv_md5 = hashlib.md5(self.obj_body).hexdigest() - if self.check_sum and recv_md5 != self.obj_md5: - # when needs check md5 and check no pass, indicates received data error, send b"ERROR" to client. - logger.debug(f"Error:接收数据有问题，流水号{self.sequence_number}, expected {self.obj_md5}, but get {recv_md5}") + if self.check_sum and recv_crc != self.obj_crc: + # when needs check hash value and check no pass, indicates received data error, send b"ERROR" to client. + logger.debug(f"Error:接收数据有问题，流水号{self.sequence_number}, expected {self.obj_crc}, but get {recv_crc}") self.send_ack(self.ACK_ERROR) else: if self.obj_body == self.ACK_STOP: @@ -146,7 +145,7 @@ class ServerProtocol(protocol.Protocol): if obj_key in self.sequence_number_dict: logger.debug(f"这是一次异常的重传，可以忽略。 {obj_key}, {self.sequence_number_dict}") else: - self.sequence_number_dict[obj_key] = self.obj_md5 + self.sequence_number_dict[obj_key] = self.obj_crc self.consumer_queue.put(self.obj_body, block=True) self.reset_env() @@ -173,7 +172,7 @@ class ServerProtocol(protocol.Protocol): self.sequence_number = -1 self.rank = -1 self.step = -1 - self.obj_md5 = None + self.obj_crc = None self.obj_body = None def dataReceived(self, data): @@ -192,15 +191,15 @@ class ServerProtocol(protocol.Protocol): logger.debug( f"流水号: {self.sequence_number}; RANK: {self.rank}; STEP: {self.step}; Length: {self.obj_length}") - # If needs check md5 but not parse md5 yet, read 32b md5 values - check_sum_and_md5 = (self.check_sum + # If needs check hash but not parse crc yet, read 8b crc values + check_sum_and_crc = (self.check_sum and self.obj_length is not None - and self.obj_md5 is None - and len(self.buffer.getvalue()) - self.tell >= self.md5_width) - if check_sum_and_md5: - self.obj_md5 = self.buffer.read(self.md5_width).decode() - self.tell += self.md5_width - logger.debug(f"MD5: {self.obj_md5}") + and self.obj_crc is None + and len(self.buffer.getvalue()) - self.tell >= self.crc_width) + if check_sum_and_crc: + self.obj_crc = self.buffer.read(self.crc_width).decode() + self.tell += self.crc_width + logger.debug(f"Hash value: {self.obj_crc}") current_length = len(self.buffer.getvalue()) - self.tell if self.obj_length is not None and 0 < self.obj_length <= current_length: diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/moe_gating_top_k_softmax.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/moe_gating_top_k_softmax.py index be15935ce9c9f77bc0a8447902f7f4a7b536a7fb..07655ba841120a80f64a9975a74abd7556569a41 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/moe_gating_top_k_softmax.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/moe_gating_top_k_softmax.py @@ -29,6 +29,8 @@ def softmax_func(x, axis=None): def npu_moe_gating_top_k_softmax(x, finished_optional, k): input_dtype = x.dtype + if x.dim() < 1: + raise ValueError("Input x must have at least 1 dimensions.") num_expert = x.shape[-1] softmax = softmax_func(x, -1) softmax = softmax.to(input_dtype) @@ -36,9 +38,13 @@ def npu_moe_gating_top_k_softmax(x, finished_optional, k): expert_idx = expert_idx[:, :k] y = torch.gather(softmax, index=expert_idx, dim=-1) if finished_optional is not None: + if finished_optional.dim() < 1: + raise ValueError("Finished_optional must have at least 1 dimensions.") finished_optional = finished_optional.view(finished_optional.shape[0], 1) finished_optional = finished_optional.expand(-1, k) expert_idx = torch.where(finished_optional, num_expert, expert_idx) + if y.dim() < 2: + raise ValueError("Variable y must have at least 2 dimensions.") row_idx = torch.arange(y.shape[0] * y.shape[1]).reshape(y.shape[1], y.shape[0]).t() return y, expert_idx, row_idx diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index 58a585f5a05f4b2d533d150db3a9fbfd907f5a07..3cdb4f6c0c7c5ac5f01c41e06905b731cd809029 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -117,6 +117,12 @@ def fusion_attention_forward(forward_params): pse = forward_params.pse scale = forward_params.scale keep_prob = forward_params.keep_prob + + # 除零风险拦截：keep_prob 为 0 时会导致除零错误 + if keep_prob == 0: + raise ValueError("fusion_attention_forward: keep_prob cannot be zero to avoid division by zero.") + + qk = calculate_qk(q, k, atten_mask, pse, scale) softmax_res, softmax_max, softmax_sum = softmax_forward(qk) if drop_mask is None or len(drop_mask.shape) == 0: @@ -137,6 +143,11 @@ def fusion_attention_backward(backward_params): pse = backward_params.pse scale = backward_params.scale keep_prob = backward_params.keep_prob + + # 除零风险拦截：keep_prob 为 0 时会导致除零错误 + if keep_prob == 0: + raise ValueError("fusion_attention_backward: keep_prob cannot be zero to avoid division by zero.") + dp = torch.matmul(dx, v.permute(0, 1, 3, 2)) if drop_mask is None or len(drop_mask.shape) == 0: drop_res = softmax_res.permute(0, 1, 3, 2) @@ -164,23 +175,35 @@ def parse_bsnd_args(query, key, head_num, input_layout): if input_layout == "BSH": b, s1, h1 = query.shape _, s2, h2 = key.shape + if n1 == 0: + raise ValueError("parse_bsnd_args: head_num (n1) cannot be zero to avoid division by zero.") d = h1 // n1 + if d == 0: + raise ValueError("parse_bsnd_args: computed head dimension (d) is zero, division by zero risk.") n2 = h2 // d elif input_layout == "SBH": s1, b, h1 = query.shape s2, _, h2 = key.shape + if n1 == 0: + raise ValueError("parse_bsnd_args: head_num (n1) cannot be zero to avoid division by zero.") d = h1 // n1 + if d == 0: + raise ValueError("parse_bsnd_args: computed head dimension (d) is zero, division by zero risk.") n2 = h2 // d elif input_layout == "BSND": b, s1, n1, d = query.shape _, s2, n2, _ = key.shape h1 = n1 * d h2 = n2 * d + if d == 0: + raise ValueError("parse_bsnd_args: head dimension (d) is zero, division by zero risk.") elif input_layout == "BNSD": b, n1, s1, d = query.shape _, n2, s2, _ = key.shape h1 = n1 * d h2 = n2 * d + if d == 0: + raise ValueError("parse_bsnd_args: head dimension (d) is zero, division by zero risk.") except Exception as e: raise ValueError(f"query.shape: {query.shape}, key.shape: {key.shape}, parse_bsnd_args error: {e}") from e @@ -446,6 +469,8 @@ def npu_fusion_attention_forward_patch(*args, **kwargs): input_layout = get_input_layout(*args, **kwargs) b, s1, s2, n1, n2, d, h1, h2, dtype = parse_bsnd_args(args[0], args[1], head_num, input_layout) + if d == 0: + raise ValueError("npu_fusion_attention_forward_patch: head dimension (d) is zero, division by zero risk.") if n1 == n2 and s1 == s2: logger.debug(f"running case : BNSD = {b}_{n1}_{s1}_{d}, sparse = {kwargs.get('sparse_mode', 0)}") else: @@ -478,6 +503,8 @@ def npu_fusion_attention_backward_patch(*args, **kwargs): raise ValueError(f"Unsupported npu_fusion_attention_grad args {args}.") b, s1, s2, n1, n2, d, h1, h2, dtype = parse_bsnd_args(args[0], args[1], args[4], args[5]) + if d == 0: + raise ValueError("npu_fusion_attention_backward_patch: head dimension (d) is zero, division by zero risk.") if n1 == n2 and s1 == s2: logger.info(f"running case : bnsd = {b}_{n1}_{s1}_{d}, sparse = {kwargs.get('sparse_mode', 0)}") else: diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 2191e545287696e9aff9b46f8f60fd3c02159f5e..e2e64eb90f634171c413528c7eeedf0e556a21f9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -24,11 +24,12 @@ from functools import wraps import numpy as np import torch import torch.distributed as dist + from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.file_utils import (FileCheckConst, change_mode, check_file_or_directory_path, check_path_before_create, FileOpen) from msprobe.core.common.log import logger -from msprobe.core.common.utils import check_seed_all +from msprobe.core.common.utils import check_seed_all, is_save_variable_valid from packaging import version try: @@ -38,7 +39,9 @@ except ImportError: else: is_gpu = False + torch_without_guard_version = torch.__version__ >= '2.1' +torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' if not is_gpu and not torch_without_guard_version: from torch_npu.utils.device_guard import torch_device_guard as torch_npu_device_guard @@ -313,14 +316,14 @@ def print_rank_0(message): logger.info(message) -def load_pt(pt_path, to_cpu=False): +def load_pt(pt_path, to_cpu=False, weights_only=True): pt_path = os.path.realpath(pt_path) check_file_or_directory_path(pt_path) try: if to_cpu: - pt = torch.load(pt_path, map_location=torch.device("cpu"), weights_only=True) + pt = torch.load(pt_path, map_location=torch.device("cpu"), weights_only=weights_only) else: - pt = torch.load(pt_path, weights_only=True) + pt = torch.load(pt_path, weights_only=weights_only) except Exception as e: raise RuntimeError(f"load pt file {pt_path} failed") from e return pt @@ -395,7 +398,7 @@ def save_api_data(api_data): io_buff = io.BytesIO() torch.save(api_data, io_buff) except Exception as e: - raise RuntimeError(f"save api_data to io_buff failed") from e + raise RuntimeError("save api_data to io_buff failed") from e return io_buff @@ -405,7 +408,7 @@ def load_api_data(api_data_bytes): buffer = io.BytesIO(api_data_bytes) buffer = torch.load(buffer, map_location="cpu") except Exception as e: - raise RuntimeError(f"load api_data from bytes failed") from e + raise RuntimeError("load api_data from bytes failed") from e return buffer @@ -423,7 +426,11 @@ def is_recomputation(): bool: True if in the re-computation phase, False otherwise. """ backward_function_indices = [] - call_stack = inspect.stack() + try: + call_stack = inspect.stack() + except Exception as e: + logger.warning(f"Failed to capture stack trace, recomputation validation may be incorrect, error info: {e}.") + return False # Identify the function 'backward' is being executed within the 'torch/_tensor.py' file. for frame_info in call_stack: @@ -453,9 +460,11 @@ def is_recomputation(): def check_save_param(variable, name, save_backward): # try catch this api to skip invalid call - if not isinstance(variable, (list, dict, tuple, torch.Tensor, int, float, str)): + valid_data_types = (torch.Tensor, int, float, str) + if not is_save_variable_valid(variable, valid_data_types): + valid_data_types_with_nested_types = valid_data_types + (dict, tuple, list) logger.warning("PrecisionDebugger.save variable type not valid, " - "should be one of list, dict, tuple, torch.Tensor, int, float or string. " + f"should be one of {valid_data_types_with_nested_types}" "Skip current save process.") raise ValueError if not isinstance(name, str): @@ -470,13 +479,8 @@ def check_save_param(variable, name, save_backward): raise ValueError -def replace_last_occurrence(text, old, new): - if text is None: - return text - index = text.rfind(old) - if index != -1: - return text[:index] + text[index:].replace(old, new, 1) - return text +def is_torch_nn_module(variable): + return isinstance(variable, torch.nn.Module) and not isinstance(variable, torch.jit.ScriptModule) def is_hifloat8_tensor(tensor): @@ -489,3 +493,17 @@ def is_float8_tensor(tensor): if str(tensor.dtype) in [Const.FLOAT8_E5M2_TYPE, Const.FLOAT8_E4M3FN_TYPE]: return True return is_hifloat8_tensor(tensor) + + +def register_forward_pre_hook(module, forward_pre_hook): + if torch_version_above_or_equal_2: + module.register_forward_pre_hook(forward_pre_hook, with_kwargs=True) + else: + module.register_forward_pre_hook(forward_pre_hook) + + +def register_forward_hook(module, forward_hook): + if torch_version_above_or_equal_2: + module.register_forward_hook(forward_hook, with_kwargs=True) + else: + module.register_forward_hook(forward_hook) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index de62af421b5a37e39140a9836fb16853443740d7..6f8ad5cf60924581f9c112e1cb236f51f255a1dd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,41 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - -from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import create_directory -from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, get_dump_mode, \ - set_dump_path -from msprobe.core.compare.acc_compare import ModeConfig -from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json, set_stack_json_path -from msprobe.pytorch.common.log import logger -from msprobe.pytorch.compare.pt_compare import PTComparator, compare +from msprobe.core.compare.utils import compare_distributed_inner +from msprobe.pytorch.compare.pt_compare import compare def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): - if kwargs.get("suffix"): - logger.error("Argument 'suffix' is not supported for compare_distributed.") - raise CompareException(CompareException.INVALID_PARAM_ERROR) - is_print_compare_log = kwargs.get("is_print_compare_log", True) - # get the ranks and match by order - npu_ranks = sorted(check_and_return_dir_contents(npu_dump_dir, 'rank')) - bench_ranks = sorted(check_and_return_dir_contents(bench_dump_dir, 'rank')) - if len(npu_ranks) != len(bench_ranks): - logger.error( - "The number of ranks in the two runs are different. " - "Unable to match the ranks. " - "Please use another folder to compare or use compare() api and manually match the ranks.") - raise CompareException(CompareException.INVALID_PATH_ERROR) - for nr, br in zip(npu_ranks, bench_ranks): - npu_data_dir = os.path.join(npu_dump_dir, nr) - bench_data_dir = os.path.join(bench_dump_dir, br) - npu_path = extract_json(npu_data_dir, stack_json=False) - bench_path = extract_json(bench_data_dir, stack_json=False) - - dump_result_param = { - "npu_json_path": npu_path, - "bench_json_path": bench_path, - "is_print_compare_log": is_print_compare_log - } - compare(input_param=dump_result_param, output_path=output_path, suffix=f'_{nr}-{br}', **kwargs) + compare_distributed_inner(npu_dump_dir, bench_dump_dir, output_path, compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 308a82b3d6e9beb67a669ea05b83d7b8a6eddc90..96e9fc88e8aa3457b44b2011732738e0d4689887 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,92 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os.path +from msprobe.core.compare.acc_compare import Comparator, ModeConfig, MappingConfig, setup_comparison +from msprobe.pytorch.compare.utils import read_pt_data -import torch -from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import FileChecker, create_directory, load_yaml -from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, get_dump_mode, \ - set_dump_path -from msprobe.core.compare.acc_compare import Comparator, ModeConfig -from msprobe.core.compare.utils import set_stack_json_path -from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import load_pt - - -class PTComparator(Comparator): - def __init__(self, mode_config, data_mapping=None): - super().__init__(mode_config) - - self.stack_mode = mode_config.stack_mode - self.auto_analyze = mode_config.auto_analyze - self.fuzzy_match = mode_config.fuzzy_match - self.dump_mode = mode_config.dump_mode - - self.frame_name = PTComparator.__name__ - self.data_mapping = data_mapping - if isinstance(self.data_mapping, str) or self.data_mapping is None: - self.data_mapping_dict = self.load_mapping_file(self.data_mapping) - elif isinstance(self.data_mapping, dict): - self.data_mapping_dict = self.data_mapping - else: - raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " - f"{type(self.data_mapping)}") - - @staticmethod - def load_mapping_file(mapping_file): - if isinstance(mapping_file, str): - mapping_dict = load_yaml(mapping_file) - else: - mapping_dict = {} - return mapping_dict - - def read_npy_data(self, dir_path, file_name): - if not file_name: - return None - data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.PT_SUFFIX, False) - data_path = path_checker.common_check() - try: - # detach because numpy can not process gradient information - data_value = load_pt(data_path, to_cpu=True).detach() - except RuntimeError as e: - # 这里捕获 load_pt 中抛出的异常 - logger.error(f"Failed to load the .pt file at {data_path}.") - raise CompareException(CompareException.INVALID_FILE_ERROR) from e - except AttributeError as e: - # 这里捕获 detach 方法抛出的异常 - logger.error(f"Failed to detach the loaded tensor.") - raise CompareException(CompareException.DETACH_ERROR) from e - if data_value.dtype == torch.bfloat16: - data_value = data_value.to(torch.float32) - data_value = data_value.numpy() - return data_value +def read_real_data(npu_dir, npu_data_name, bench_dir, bench_data_name, _) -> tuple: + n_value = read_pt_data(npu_dir, npu_data_name) + b_value = read_pt_data(bench_dir, bench_data_name) + return n_value, b_value def compare(input_param, output_path, **kwargs): - try: - auto_analyze = kwargs.get('auto_analyze', True) - fuzzy_match = kwargs.get('fuzzy_match', False) - data_mapping = kwargs.get('data_mapping', None) - suffix = kwargs.get('suffix', '') - - set_dump_path(input_param) - dump_mode = get_dump_mode(input_param) - if "stack_json_path" in input_param: - stack_mode = kwargs.get('stack_mode', False) - else: - stack_mode = set_stack_json_path(input_param) # set stack_mode and set "stack_json_path" in input_param - check_configuration_param(stack_mode, auto_analyze, fuzzy_match, input_param.get('is_print_compare_log', True)) - create_directory(output_path) - check_compare_param(input_param, output_path, dump_mode, stack_mode) - except (CompareException, FileCheckException) as error: - logger.error('Compare failed. Please check the arguments and do it again!') - raise CompareException(error.code) from error + config = setup_comparison(input_param, output_path, **kwargs) - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - pt_comparator = PTComparator(mode_config, data_mapping) - pt_comparator.compare_core(input_param, output_path, suffix=suffix) + mode_config = ModeConfig(config.stack_mode, config.auto_analyze, config.fuzzy_match, + config.dump_mode, config.compared_file_type) + mapping_config = MappingConfig(data_mapping=config.data_mapping) + pt_comparator = Comparator(read_real_data, mode_config, mapping_config) + pt_comparator.compare_core(input_param, output_path, suffix=config.suffix) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/utils.py b/debug/accuracy_tools/msprobe/pytorch/compare/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..16473ff386d89de5f3bbb269e69837c07a950ea5 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/utils.py @@ -0,0 +1,47 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import torch + +from msprobe.core.common.utils import logger, CompareException +from msprobe.core.common.file_utils import FileChecker, FileCheckConst +from msprobe.pytorch.common.utils import load_pt + + +def read_pt_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.PT_SUFFIX, False) + data_path = path_checker.common_check() + try: + # detach because numpy can not process gradient information + data_value = load_pt(data_path, to_cpu=True).detach() + except RuntimeError as e: + # 这里捕获 load_pt 中抛出的异常 + logger.error(f"Failed to load the .pt file at {data_path}.") + raise CompareException(CompareException.INVALID_FILE_ERROR) from e + except AttributeError as e: + # 这里捕获 detach 方法抛出的异常 + logger.error(f"Failed to detach the loaded tensor.") + raise CompareException(CompareException.DETACH_ERROR) from e + if data_value.dtype == torch.bfloat16: + data_value = data_value.to(torch.float32) + data_value = data_value.numpy() + return data_value diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index 77e78bc38063602e64b533291d60b9b12fd2ae00..5d678880f4288e3a4f9349b5b66689c7309ba2ee 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -13,11 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import torch - from msprobe.core.common.const import Const from msprobe.core.common.exceptions import MsprobeException from msprobe.pytorch.common.log import logger +from msprobe.pytorch.common.utils import is_torch_nn_module class DebuggerConfig: @@ -60,6 +59,7 @@ class DebuggerConfig: if isinstance(task_config.online_run_ut_recompute, bool) else False self.check() + self._check_statistics_config(task_config) if self.level == Const.LEVEL_L2: self.is_backward_kernel_dump = False @@ -78,10 +78,13 @@ class DebuggerConfig: if not isinstance(self.async_dump, bool): raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, f"The parameters async_dump should be bool.") - if self.async_dump and self.task == Const.TENSOR and not self.list: - raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, - f"The parameters async_dump is true in tensor task, the parameters list cannot be " - f"empty.") + if self.async_dump and self.task == Const.TENSOR: + if self.level == Const.LEVEL_DEBUG: + self.list = [] # async_dump + debug level case ignore list + if not self.list and self.level != Const.LEVEL_DEBUG: + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, + f"The parameters async_dump is true in tensor task, the parameters list cannot be " + f"empty.") if self.task == Const.STRUCTURE and self.level not in [Const.LEVEL_L0, Const.LEVEL_MIX]: logger.warning_on_rank_0( f"When the task is set to structure, the level should be one of {[Const.LEVEL_L0, Const.LEVEL_MIX]}. " @@ -93,25 +96,28 @@ class DebuggerConfig: self.check_kwargs() return True - def check_model(self, instance, start_model): - if self.level not in [Const.LEVEL_L0, Const.LEVEL_MIX]: - if instance.model is not None or start_model is not None: + def check_model(self, instance, start_model, token_range=None): + instance.model = start_model if start_model is not None else instance.model + if self.level not in [Const.LEVEL_L0, Const.LEVEL_MIX] and token_range is None: + if instance.model is not None: logger.info_on_rank_0( - f"The current level is not L0 or mix level, so the model parameters will not be used.") + f"The current level is not L0 or mix level and token_range is None, " + f"so the model parameter will not be used") return - if start_model is None and instance.model is None: + + if instance.model is None: logger.error_on_rank_0( - f"For level {self.level}, PrecisionDebugger or start interface must receive a 'model' parameter.") + f"For level {self.level} or non-empty token_range, " + f"PrecisionDebugger or start interface must receive a 'model' parameter.") raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, f"missing the parameter 'model'") - instance.model = start_model if start_model is not None else instance.model - if isinstance(instance.model, torch.nn.Module): + if is_torch_nn_module(instance.model): return error_model = None if isinstance(instance.model, (list, tuple)): for model in instance.model: - if not isinstance(model, torch.nn.Module): + if not is_torch_nn_module(model): error_model = model break else: @@ -119,7 +125,7 @@ class DebuggerConfig: if error_model is not None: error_info = (f"The 'model' parameter must be a torch.nn.Module or list[torch.nn.Module] " - f"type, currently there is a {type(error_model)} type.") + f"type, currently there is an unsupported {type(error_model)} type.") raise MsprobeException( MsprobeException.INVALID_PARAM_ERROR, error_info) @@ -130,8 +136,23 @@ class DebuggerConfig: if not self.list or len(self.list) != 1: raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, f"When level is set to L2, the list must be configured as a list with one api name.") + if self.task != Const.TENSOR: + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, + f"When level is set to L2, the task must be set to tensor.") + api_name = self.list[0] if api_name.endswith(Const.BACKWARD): self.is_backward_kernel_dump = True api_forward_name = api_name[:-len(Const.BACKWARD)] + Const.FORWARD self.list.append(api_forward_name) + + def _check_statistics_config(self, task_config): + if self.task != Const.STATISTICS: + return + self.tensor_list = [] + if not hasattr(task_config, "tensor_list"): + return + if self.level == Const.LEVEL_DEBUG and task_config.tensor_list: + logger.warning_on_rank_0("When level is set to debug, the tensor_list will be invalid.") + return + self.tensor_list = task_config.tensor_list diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py index e6b014e528463ccae248ba01c3cca7be782aaaaf..a3c5b7fb25ac91a45845ab4459da171cd6ff9e69 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py @@ -12,22 +12,22 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import functools from collections import namedtuple -import torch +from torch.utils.data import dataloader + from msprobe.core.common.const import Const, FileCheckConst, MsgConst from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common.file_utils import FileChecker -from msprobe.core.common.utils import get_real_step_or_rank, check_init_step +from msprobe.core.common.utils import get_real_step_or_rank, check_init_step, check_token_range from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import check_save_param +from msprobe.pytorch.common.utils import check_save_param, is_torch_nn_module from msprobe.pytorch.debugger.debugger_config import DebuggerConfig from msprobe.pytorch.dump.module_dump.module_dump import ModuleDumper from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor from msprobe.pytorch.pt_config import parse_json_config from msprobe.pytorch.service import Service -from torch.utils.data import dataloader ConfigParameters = namedtuple("ConfigParameters", ["config_path", "task", "dump_path", "level", "model"]) @@ -76,6 +76,7 @@ class PrecisionDebugger: self.service = Service(self.config) self.module_dumper = ModuleDumper(self.service) self.enable_dataloader = self.config.enable_dataloader + self.ori_customer_func = {} if self.enable_dataloader: logger.warning_on_rank_0("The enable_dataloader feature will be deprecated in the future.") dataloader._BaseDataLoaderIter.__next__ = iter_tracer(dataloader._BaseDataLoaderIter.__next__) @@ -114,17 +115,20 @@ class PrecisionDebugger: ) @classmethod - def start(cls, model=None): + def start(cls, model=None, token_range=None): instance = cls._instance if not instance: raise Exception(MsgConst.NOT_CREATED_INSTANCE) if instance.task in PrecisionDebugger.tasks_not_need_debugger: return - instance.config.check_model(instance, model) + + check_token_range(token_range) + instance.config.check_model(instance, model, token_range) + if instance.enable_dataloader: logger.warning_on_rank_0("DataLoader is enabled, start() skipped.") else: - instance.service.start(instance.model) + instance.service.start(instance.model, token_range) @classmethod def forward_backward_dump_end(cls): @@ -179,13 +183,40 @@ class PrecisionDebugger: raise Exception(MsgConst.NOT_CREATED_INSTANCE) check_init_step(step) instance.service.init_step = step + instance.service.loop = 0 + + @classmethod + def register_custom_api(cls, module, api, api_prefix=None): + if not api_prefix: + api_prefix = getattr(module, "__name__", "Custom") + if not isinstance(api_prefix, str): + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, "api_prefix must be string") + if not hasattr(module, api): + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, f"module {str(module)} does not have {api}") + instance = cls._instance + if not instance: + raise Exception(MsgConst.NOT_CREATED_INSTANCE) + instance.service.register_custom_api(module, api, api_prefix) + + @classmethod + def restore_custom_api(cls, module, api): + if not hasattr(module, api): + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, f"module {str(module)} does not have {api}") + instance = cls._instance + if not instance: + raise Exception(MsgConst.NOT_CREATED_INSTANCE) + instance.service.restore_custom_api(module, api) def module_dump(module, dump_name): - if not isinstance(module, torch.nn.Module): + if not is_torch_nn_module(module): raise MsprobeException( MsprobeException.INVALID_PARAM_ERROR, - f"the module argument in module_dump must be a torch.nn.Module subclass" + f"the module argument in module_dump must be a torch.nn.Module type, " + f"but currently there is an unsupported {type(module)} type." ) if not isinstance(dump_name, str): raise MsprobeException( diff --git a/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/hook_wrapper.py b/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/hook_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..0434e3e62686ac0f8011ea8e58daadd9da81c3c0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/hook_wrapper.py @@ -0,0 +1,93 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import wraps + +import torch +from torch.utils.hooks import BackwardHook + +from msprobe.core.common.const import Const +from msprobe.core.common.decorator import recursion_depth_decorator +from msprobe.pytorch.common.log import logger +from msprobe.pytorch.common.utils import is_float8_tensor + + +def wrap_setup_backward_hook(func): + def requires_clone(tensor): + return isinstance(tensor, torch.Tensor) and not is_float8_tensor(tensor) and \ + tensor.requires_grad and torch.is_grad_enabled() + + @recursion_depth_decorator("Dump: wrap_setup_backward_hook.parse_tensor", max_depth=Const.DUMP_MAX_DEPTH) + def parse_tensor(item, tensor_list): + if requires_clone(item): + tensor_list.append(item) + elif isinstance(item, (list, tuple)): + for value in item: + parse_tensor(value, tensor_list) + elif isinstance(item, dict): + for value in item.values(): + parse_tensor(value, tensor_list) + + @recursion_depth_decorator("Dump: wrap_setup_backward_hook.rebuild_args", max_depth=Const.DUMP_MAX_DEPTH) + def rebuild_args(item, tensor_iter): + if requires_clone(item): + result = next(tensor_iter) + if hasattr(result, "_base") and result._base is not None: + if torch._C._autograd._get_creation_meta(result) != torch._C._autograd.CreationMeta(0): + torch._C._autograd._set_creation_meta(result, torch._C._autograd.CreationMeta(0)) + return result + if isinstance(item, list): + for index, value in enumerate(item): + item[index] = rebuild_args(value, tensor_iter) + return item + if isinstance(item, dict): + for key, value in item.items(): + item[key] = rebuild_args(value, tensor_iter) + return item + if isinstance(item, tuple): + if hasattr(item, '_fields'): + return type(item)(*[rebuild_args(i, tensor_iter) for i in item]) + return type(item)([rebuild_args(i, tensor_iter) for i in item]) + return item + + @wraps(func) + def wrap_setup_hook_func(*args, **kwargs): + if len(args) < 2: + return func(*args, **kwargs) + + actual_args = args[1] + + tensor_list = [] + + parse_tensor(actual_args, tensor_list) + + new_args = args[0], tuple(tensor_list) + hooked_tensors = func(*new_args, **kwargs) + + tensor_iter = iter(hooked_tensors) + try: + new_data = rebuild_args(actual_args, tensor_iter) + except Exception as e: + logger.debug(f"Unsupported data in setup input/output hook. The detail info: {e}") + new_data = actual_args + + return new_data + + return wrap_setup_hook_func + + +def wrap_setup_input_output_hook(): + BackwardHook.setup_input_hook = wrap_setup_backward_hook(BackwardHook.setup_input_hook) + BackwardHook.setup_output_hook = wrap_setup_backward_hook(BackwardHook.setup_output_hook) diff --git a/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/module_dump.py b/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/module_dump.py index cc78962f401a9e4f46d5794d7ca074f2e37f45e0..5bf26f7ac0d91cce630a3b9c8e648453ae4ab65c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/module_dump.py +++ b/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/module_dump.py @@ -13,75 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -import torch -from msprobe.core.common.const import Const -from msprobe.core.data_dump.scope import BaseScope from msprobe.pytorch.common.log import logger +from msprobe.pytorch.dump.module_dump.module_processer import ModuleProcesser from msprobe.pytorch.hook_module.api_register import get_api_register -torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' - class ModuleDumper: def __init__(self, service): self.service = service - self.hook_handle_list = [] self.api_register = get_api_register() def start_module_dump(self, module, dump_name): + if hasattr(module, 'msprobe_hook') and not hasattr(module, 'msprobe_module_dump'): + logger.info_on_rank_0("The init dump is enabled, and the module dump function will not be available.") + return + + ModuleProcesser.enable_module_dump = True self.api_register.restore_all_api() - self.register_hook(module, dump_name) + if not hasattr(module, 'msprobe_module_dump'): + self.service.module_processor.register_module_hook(module, self.service.build_hook, + recursive=False, module_names=[dump_name]) + setattr(module, 'msprobe_module_dump', True) def stop_module_dump(self): + ModuleProcesser.enable_module_dump = False self.api_register.register_all_api() - for hook_handle in self.hook_handle_list: - if isinstance(hook_handle, torch.utils.hooks.RemovableHandle): - hook_handle.remove() - self.hook_handle_list.clear() - - def register_hook(self, module, dump_name): - prefix_name = ( - BaseScope.Module_Type_Module + Const.SEP + - dump_name + Const.SEP + - module.__class__.__name__ + Const.SEP - ) - module_processor = self.service.module_processor - _, forward_hook, backward_hook, forward_hook_torch_version_below_2 = self.service.build_hook( - BaseScope.Module_Type_Module, - prefix_name - ) - - if module_processor.has_register_backward_hook(module): - logger.warning( - f"The {dump_name} module has registered deprecated register_backward_hook," - f"which may cause abnormal data dump. The backward data dump for this module will be skipped." - ) - if torch_version_above_or_equal_2: - forward_hook_handle = module.register_forward_hook(forward_hook, with_kwargs=True) - else: - if not module_processor.has_register_backward_hook(module): - backward_hook_handle = module.register_full_backward_hook( - module_processor.node_hook(prefix_name + Const.BACKWARD, Const.STOP) - ) - self.hook_handle_list.append(backward_hook_handle) - forward_hook_handle = module.register_forward_hook(forward_hook_torch_version_below_2) - self.hook_handle_list.append(forward_hook_handle) - if not module_processor.has_register_backward_hook(module): - backward_hook_handle = module.register_full_backward_hook(backward_hook) - self.hook_handle_list.append(backward_hook_handle) - - forward_pre_hook_handle = module.register_forward_pre_hook( - module_processor.node_hook(prefix_name + Const.FORWARD, Const.START) - ) - forward_hook_handle = module.register_forward_hook( - module_processor.node_hook(prefix_name + Const.FORWARD, Const.STOP) - ) - self.hook_handle_list.extend([forward_pre_hook_handle, forward_hook_handle]) - if torch_version_above_or_equal_2 and not module_processor.has_register_backward_hook(module): - backward_pre_hook_handle = module.register_full_backward_pre_hook( - module_processor.node_hook(prefix_name + Const.BACKWARD, Const.START) - ) - backward_hook_handle = module.register_full_backward_hook( - module_processor.node_hook(prefix_name + Const.BACKWARD, Const.STOP) - ) - self.hook_handle_list.extend([backward_pre_hook_handle, backward_hook_handle]) diff --git a/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/module_processer.py index 37611f4db3238b0002f4a2f69ea94f98f38c09e2..01ea4ee664a8b43d008a6b04cbf4bbd22e125a2e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/dump/module_dump/module_processer.py @@ -13,19 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from functools import wraps +from collections import OrderedDict import torch +from torch.utils.hooks import BackwardHook, RemovableHandle + from msprobe.core.common.const import Const -from msprobe.core.common.utils import recursion_depth_decorator from msprobe.core.data_dump.scope import BaseScope, ModuleRangeScope, MixRangeScope from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import replace_last_occurrence, is_float8_tensor -from torch.utils.checkpoint import checkpoint as origin_checkpoint -from torch.utils.checkpoint import set_checkpoint_early_stop -from torch.utils.hooks import BackwardHook +from msprobe.pytorch.common.utils import is_torch_nn_module, register_forward_pre_hook +from msprobe.pytorch.dump.module_dump.hook_wrapper import wrap_setup_input_output_hook torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' +if torch_version_above_or_equal_2: + from torch.utils.checkpoint import checkpoint as origin_checkpoint, set_checkpoint_early_stop def checkpoint_without_early_stop(*args, **kwargs): @@ -34,7 +35,18 @@ def checkpoint_without_early_stop(*args, **kwargs): def replace_checkpoint(): - torch.utils.checkpoint.checkpoint = checkpoint_without_early_stop + if torch_version_above_or_equal_2: + torch.utils.checkpoint.checkpoint = checkpoint_without_early_stop + + +def wrap_megatron_deallocate(func): + def wrapper_func(out, deallocate_pipeline_outputs=False): + if deallocate_pipeline_outputs and isinstance(out, torch.Tensor) and getattr(out, "_base") is not None: + out_clone = out.clone() + out.data = torch.empty((1,), device=out.device, dtype=out.dtype, ) + return func(out_clone, deallocate_pipeline_outputs) + return func(out, deallocate_pipeline_outputs) + return wrapper_func class ModuleProcesser: @@ -42,38 +54,25 @@ class ModuleProcesser: module_stack = [] api_parent_node = "" module_node = {} + module_bw_hook_kernels = {} + module_with_backward_hook = {} + enable_module_dump = False def __init__(self, scope): self.scope = scope if isinstance(scope, (ModuleRangeScope, MixRangeScope)) else None - BackwardHook.setup_input_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_input_hook) - BackwardHook.setup_output_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_output_hook) + wrap_setup_input_output_hook() replace_checkpoint() + try: + from megatron.core.pipeline_parallel import schedules + schedules.deallocate_output_tensor = wrap_megatron_deallocate(schedules.deallocate_output_tensor) + logger.info_on_rank_0("Patch megatron method success.") + except ImportError: + logger.info_on_rank_0("No megatron find.") + except Exception as e: + logger.info_on_rank_0(f"Patch megatron method failed, detail:{str(e)}") @staticmethod - def clone_return_value(func): - @wraps(func) - def clone_return_value_func(*args, **kwargs): - result = func(*args, **kwargs) - return ModuleProcesser.clone_if_tensor(result) - - return clone_return_value_func - - @staticmethod - @recursion_depth_decorator("ModuleDump: ModuleProcesser.clone_if_tensor") - def clone_if_tensor(result): - if isinstance(result, torch.Tensor) and not is_float8_tensor(result): - return result.clone() - elif type(result) is tuple: - return tuple(ModuleProcesser.clone_if_tensor(x) for x in result) - elif type(result) is list: - return list(ModuleProcesser.clone_if_tensor(x) for x in result) - elif type(result) is dict: - return {k: ModuleProcesser.clone_if_tensor(v) for k, v in result.items()} - else: - return result - - @staticmethod - def module_count_func(module_name): + def set_and_get_calls_number(module_name): if module_name not in ModuleProcesser.module_count: ModuleProcesser.module_count[module_name] = 0 else: @@ -87,13 +86,19 @@ class ModuleProcesser: module._is_full_backward_hook is False @staticmethod - def get_modules_and_names(models): + def get_modules_and_names(models, recursive, module_names): modules_and_names_with_index = {} if isinstance(models, (list, tuple)): + if not recursive and len(module_names) != len(models): + return modules_and_names_with_index for index, model in enumerate(models): - modules_and_names_with_index[str(index)] = model.named_modules() + modules_and_names_with_index[str(index)] = model.named_modules() if recursive else \ + [(module_names[index], model)] else: - modules_and_names_with_index["-1"] = models.named_modules() + if not recursive and len(module_names) != 1: + return modules_and_names_with_index + modules_and_names_with_index["-1"] = models.named_modules() if recursive else \ + [(module_names[0], models)] return modules_and_names_with_index @classmethod @@ -102,107 +107,134 @@ class ModuleProcesser: cls.module_stack = [] cls.api_parent_node = "" cls.module_node = {} + cls.module_bw_hook_kernels = {} + cls.enable_module_dump = False + + def register_module_hook(self, models, build_hook, recursive=True, module_names=None): + if module_names is None: + module_names = [] - def register_module_hook(self, models, build_hook): - logger.info_on_rank_0("The init dump is enabled, and the module dump function will not be available.") - modules_and_names_with_index = self.get_modules_and_names(models) + modules_and_names_with_index = self.get_modules_and_names(models, recursive, module_names) for index, modules_and_names in modules_and_names_with_index.items(): model = models if index == "-1" else models[int(index)] for name, module in modules_and_names: - if module == model: + if recursive and module == model: + continue + if not is_torch_nn_module(module): + logger.warning( + f"The module dump does not support {type(module)} type. " + f"The data dump for this module will be skipped." + ) continue if module.__class__.__name__ == "FullyShardedDataParallel": continue + setattr(module, 'msprobe_hook', True) module_index = (index + Const.SEP) if index != "-1" else "" - prefix_name = (BaseScope.Module_Type_Module + Const.SEP + module_index + - name + Const.SEP + module.__class__.__name__ + Const.SEP) - pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 = build_hook( - BaseScope.Module_Type_Module, - prefix_name - ) + prefix_name = f'{BaseScope.Module_Type_Module}{Const.SEP}{module_index}{name}{Const.SEP}' + \ + f'{module.__class__.__name__}{Const.SEP}' + + forward_pre_hook = self.build_module_hook(prefix_name, build_hook) if self.has_register_backward_hook(module): logger.warning( f"The {prefix_name[:-1]} has registered deprecated register_backward_hook," f"which may cause abnormal data dump. The backward data dump for this module will be skipped." ) + ModuleProcesser.module_with_backward_hook[prefix_name] = True + register_forward_pre_hook(module, forward_pre_hook) + + def build_module_hook(self, module_name, build_data_hook): + def forward_pre_hook(module, args, kwargs=None): + if kwargs is None: + kwargs = {} + + if hasattr(module, 'msprobe_module_dump') and not self.enable_module_dump: + return (args, kwargs) if torch_version_above_or_equal_2 else args + + index = ModuleProcesser.set_and_get_calls_number(module_name) + full_forward_name = f'{module_name}{Const.FORWARD}{Const.SEP}{index}' + full_backward_name = f'{module_name}{Const.BACKWARD}{Const.SEP}{index}' + + self.set_construct_info_in_pre_hook(full_forward_name) + + if not hasattr(module, 'msprobe_forward_hook'): + forward_hooks_dict = getattr(module, '_forward_hooks', OrderedDict()) + handle = RemovableHandle(forward_hooks_dict) + forward_hooks_dict[handle.id] = forward_hook + forward_hooks_dict.move_to_end(handle.id, last=False) if torch_version_above_or_equal_2: - module.register_forward_hook(forward_hook, with_kwargs=True) + forward_hooks_with_kwargs_dict = getattr(module, '_forward_hooks_with_kwargs', OrderedDict()) + forward_hooks_with_kwargs_dict[handle.id] = True + + setattr(module, 'msprobe_forward_hook', True) + + _, _, backward_data_hook = build_data_hook(BaseScope.Module_Type_Module, full_forward_name) + + def get_backward_pre_hook(full_backward_name): + def backward_pre_hook_fn(module, grad_output): + self.set_construct_info_in_pre_hook(full_backward_name) + return backward_pre_hook_fn + + def get_backward_hook(backward_data_hook, full_backward_name): + def backward_hook_fn(module, grad_input, grad_output): + new_output = backward_data_hook(module, grad_input, grad_output) + self.set_construct_info_in_hook(full_backward_name, is_forward=False) + return new_output + return backward_hook_fn + + if not ModuleProcesser.module_with_backward_hook.get(module_name): + backward_pre_hook = get_backward_pre_hook(full_backward_name) + backward_hook = get_backward_hook(backward_data_hook, full_backward_name) + if torch_version_above_or_equal_2: + bw_hook = BackwardHook(module, [backward_hook], [backward_pre_hook]) else: - if not self.has_register_backward_hook(module): - module.register_full_backward_hook(self.node_hook(prefix_name + Const.BACKWARD, Const.STOP)) - module.register_forward_hook(forward_hook_torch_version_below_2) - if not self.has_register_backward_hook(module): - module.register_full_backward_hook(backward_hook) - - module.register_forward_pre_hook(self.node_hook(prefix_name + Const.FORWARD, Const.START)) - module.register_forward_hook(self.node_hook(prefix_name + Const.FORWARD, Const.STOP)) - if torch_version_above_or_equal_2 and not self.has_register_backward_hook(module): - module.register_full_backward_pre_hook(self.node_hook(prefix_name + Const.BACKWARD, Const.START)) - module.register_full_backward_hook(self.node_hook(prefix_name + Const.BACKWARD, Const.STOP)) - - def node_hook(self, name_prefix, start_or_stop, **kwargs): - - def pre_hook(module, input, output=None): - try: - index = ModuleProcesser.module_count_func(name_prefix) - except IndexError as e: - index = None - pass - full_name = name_prefix + Const.SEP + str(index) - if not hasattr(module, "mindstudio_reserved_name") or not module.mindstudio_reserved_name: - module.mindstudio_reserved_name = [] - module.mindstudio_reserved_name.append(full_name) - if self.module_stack: - ModuleProcesser.module_node[full_name] = self.module_stack[-1] + bw_hook = BackwardHook(module, [backward_hook]) + ModuleProcesser.module_bw_hook_kernels[full_forward_name] = bw_hook + args = bw_hook.setup_input_hook(args) + return (args, kwargs) if torch_version_above_or_equal_2 else args + + def forward_hook(module, args, kwargs_or_output, output_or_kwargs=None): + if hasattr(module, 'msprobe_module_dump') and not self.enable_module_dump: + return output_or_kwargs if torch_version_above_or_equal_2 else kwargs_or_output + + index = ModuleProcesser.module_count.get(module_name) + full_name = f'{module_name}{Const.FORWARD}{Const.SEP}{index}' + + _, forward_data_hook, _ = build_data_hook(BaseScope.Module_Type_Module, full_name) + hook_result = forward_data_hook(module, args, kwargs_or_output, output_or_kwargs) + self.set_construct_info_in_hook(full_name) + + if hook_result is not None: + result = hook_result else: - ModuleProcesser.module_node[full_name] = None + result = output_or_kwargs if torch_version_above_or_equal_2 else kwargs_or_output - ModuleProcesser.module_stack.append(full_name) - if self.module_stack: - ModuleProcesser.api_parent_node = self.module_stack[-1] - if self.scope: - self.scope.begin_module(full_name) + bw_hook = ModuleProcesser.module_bw_hook_kernels.get(full_name) + if bw_hook: + result = bw_hook.setup_output_hook(result) - def end_hook(module, input, output=None): + return result + + return forward_pre_hook + + def set_construct_info_in_pre_hook(self, full_name): + if self.module_stack: + ModuleProcesser.module_node[full_name] = self.module_stack[-1] + else: + ModuleProcesser.module_node[full_name] = None + ModuleProcesser.module_stack.append(full_name) + ModuleProcesser.api_parent_node = full_name + if self.scope: + self.scope.begin_module(full_name) + + def set_construct_info_in_hook(self, full_name, is_forward=True): + if torch_version_above_or_equal_2 or is_forward: if self.module_stack: ModuleProcesser.module_stack.pop() - if self.module_stack: - ModuleProcesser.api_parent_node = self.module_stack[-1] - else: - ModuleProcesser.api_parent_node = None - if not hasattr(module, "mindstudio_reserved_name") or not module.mindstudio_reserved_name: - raise RuntimeError(f"module reserve name is None when pop") - current_name = module.mindstudio_reserved_name.pop() + ModuleProcesser.api_parent_node = ModuleProcesser.module_stack[-1] if self.module_stack else None if self.scope: - self.scope.end_module(current_name) - - def backward_hook(module, input, output=None): - try: - index = ModuleProcesser.module_count_func(name_prefix) - except IndexError as e: - index = None - pass - full_name = name_prefix + Const.SEP + str(index) - if not hasattr(module, "mindstudio_reserved_name") or not module.mindstudio_reserved_name: - module.mindstudio_reserved_name = [] - module.mindstudio_reserved_name.append(full_name) - forward_full_name = replace_last_occurrence(full_name, Const.BACKWARD, Const.FORWARD) - ModuleProcesser.module_node[full_name] = replace_last_occurrence( - ModuleProcesser.module_node.get(forward_full_name), Const.FORWARD, Const.BACKWARD) - ModuleProcesser.api_parent_node = None + self.scope.end_module(full_name) + else: if self.scope: self.scope.begin_module(full_name) - - if torch_version_above_or_equal_2: - if Const.START in start_or_stop: - return pre_hook - else: - return end_hook - else: - if Const.FORWARD in name_prefix and Const.START in start_or_stop: - return pre_hook - elif Const.BACKWARD in name_prefix: - return backward_hook - else: - return end_hook + ModuleProcesser.api_parent_node = full_name diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py index a5e7cabd85186336b7b4cb5bf5d6f25599ad9d7f..fde22f7345fd5e43a86cb2037ef4beff31129fa8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py @@ -16,7 +16,7 @@ import torch from msprobe.core.common.exceptions import FreeBenchmarkException -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.free_benchmark.common.enums import DeviceType from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/single_benchmark.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/single_benchmark.py index 49e845da4011565f1b6ccf0c0e1193fb3fcffcbf..a5f18946c44c09bf1670173d45cc99ace3b0e79d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/single_benchmark.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/single_benchmark.py @@ -16,7 +16,7 @@ import math import torch -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig from msprobe.pytorch.free_benchmark.common.utils import TorchC diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py index 41ec39e3a3b6233720c047d5d2b736d91bba989e..754e3b06e9670a04fcf7c20d5af3d7e1733b7af1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py @@ -14,7 +14,7 @@ # limitations under the License. import torch -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode @@ -95,13 +95,13 @@ class AddNoiseLayer(NpuBaseLayer): except Exception: logger.warning_on_rank_0( f"[msprobe] Free Benchmark: For {self.api_name}, " - f"when calculate maximun value, tensor is changed to float32." + f"when calculating the maximum value, the tensor is changed to float32." ) max_val = TorchC.max(TorchC.abs(tensor_obj.to(torch.float32))).item() if max_val < abs_tol: logger.warning_on_rank_0( f"[msprobe] Free Benchmark: For {self.api_name}, " - f"Maximun value is less than the minimun threshold. Cancel add noise." + f"maximum value is less than the minimum threshold. Cancel adding noise." ) return False return True diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py index df1a73127aa0b69e42254cce1d3334810319f7cf..aec0c3ca96e39958316f6835261618c148c7ad4e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py @@ -14,7 +14,7 @@ # limitations under the License. import torch -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.constant import ThresholdConfig from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode @@ -100,13 +100,13 @@ class BitNoiseLayer(NpuBaseLayer): except Exception: logger.warning_on_rank_0( f"[msprobe] Free Benchmark: For {self.api_name}, " - f"when calculate maximun value, tensor is changed to float32." + f"when calculate the maximum value, the tensor is changed to float32." ) max_val = TorchC.max(TorchC.abs(tensor_obj.to(torch.float32))).item() if max_val < abs_tol: logger.warning_on_rank_0( f"[msprobe] Free Benchmark: For {self.api_name}, " - f"Maximun value is less than the minimun threshold. Cancel add noise." + f"maximum value is less than the minimum threshold. Cancel adding noise." ) return False return True diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py index c4fbeaf82f8fcafba235a7faa6dd9073d4d556d8..521637a1d8b3bca226a6eacfc5f6f5a0d4bc1921 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py @@ -14,7 +14,7 @@ # limitations under the License. import torch -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode from msprobe.pytorch.free_benchmark.common.params import DataParams diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py index 095e77ffaff39a795cb1418c1695608d91d7427b..daa271976f3b05f81b9997bd1775ee2809b776c9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py @@ -15,7 +15,7 @@ import torch from msprobe.core.common.const import Const -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.constant import CommonField from msprobe.pytorch.free_benchmark.common.enums import PerturbationMode diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/check_handler.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/check_handler.py index 9feec1531b16ff8ba63910f3f7c40aa275d0104e..d088cd1d1647a59c167f705702d9ad6afcf6e21b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/check_handler.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/check_handler.py @@ -49,6 +49,6 @@ class CheckerHandler(FuzzHandler): except Exception as e: logger.warning_on_rank_0( f"[msprobe] Free Benchmark: For {self.params.api_name}, " - f"when campare the result exception raise {e}" + f"when comparing the results, an exception is raised: {e}" ) return data_params.original_result diff --git a/debug/accuracy_tools/msprobe/pytorch/function_factory.py b/debug/accuracy_tools/msprobe/pytorch/function_factory.py index 247e2cd0ed5ea11047cc0d75954dbc1e92b889f4..f515b5d4783c0e20a2303579f6954d42a7b9deac 100644 --- a/debug/accuracy_tools/msprobe/pytorch/function_factory.py +++ b/debug/accuracy_tools/msprobe/pytorch/function_factory.py @@ -70,7 +70,7 @@ class Register(dict): def add_register_item(key, value): if key in self._dict: - logger.warning(f"{value.__name__} has been registered before, so we will overriden it.") + logger.warning(f"{value.__name__} has been registered before, so we will override it.") self[key] = value return value diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index 926476b8fb353531e54a485ccb47c4c59860c5d0..81d7575fc251c0b90703b13c537f61f778cf5136 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -46,7 +46,7 @@ class GradientMonitor: if not os.path.exists(self._output_path): create_directory(self._output_path) else: - logger.warning(f"the file in {self._output_path} will be recoverd") + logger.warning(f"the file in {self._output_path} will be deleted") self._step = -1 self._param2name = defaultdict(str) @@ -97,7 +97,7 @@ class GradientMonitor: create_directory(output_dirpath) output_path = os.path.join(output_dirpath, f"grad_summary_{self._step}.csv") if os.path.exists(output_path): - logger.warning(f"{output_path} will be recoverd") + logger.warning(f"{output_path} will be deleted") remove_path(output_path) header_result = GradStatCsv.generate_csv_header(self._level_adp, self._bounds) output_lines.insert(0, header_result) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/api_register.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/api_register.py index 30a45a84d877b5b1dd4b4d0231faece010c59f61..7ef5622641d42b507aed95631d248418b92049a2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/api_register.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/api_register.py @@ -15,21 +15,36 @@ import functools import os +import inspect import torch import torch.distributed as dist from msprobe.core.common.const import Const from msprobe.core.data_dump.api_registry import ApiRegistry +from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.utils import ( torch_without_guard_version, is_gpu, torch_device_guard, parameter_adapter ) from msprobe.pytorch.function_factory import npu_custom_functions from msprobe.pytorch.hook_module.hook_module import HOOKModule +from msprobe.pytorch.hook_module.utils import dynamic_import_op +from msprobe.core.common.file_utils import load_yaml + +try: + import mindspeed.ops +except ImportError: + mindspeed_enable = False +else: + mindspeed_enable = True torch_version_above_2 = torch.__version__.split('+')[0] > '2.0' +_inner_used_api = {} +_supported_api_list_path = (os.path.join(os.path.dirname(os.path.realpath(__file__)), Const.SUPPORT_API_FILE_NAME),) +_cuda_func_mapping = {"npu_fusion_attention": "gpu_fusion_attention"} + _api_types = { Const.PT_FRAMEWORK: { Const.PT_API_TYPE_FUNCTIONAL: (torch.nn.functional, (torch.nn.functional,)), @@ -57,10 +72,11 @@ if not is_gpu: torch_npu.distributed.distributed_c10d)) } ) - -_inner_used_api = {} -_supported_api_list_path = (os.path.join(os.path.dirname(os.path.realpath(__file__)), Const.SUPPORT_API_FILE_NAME),) -_cuda_func_mapping = {"npu_fusion_attention": "gpu_fusion_attention"} + if mindspeed_enable: + _api_types.get(Const.PT_FRAMEWORK).update({Const.PT_API_TYPE_MINDSPEED: (mindspeed.ops, (mindspeed.ops,))}) + mindspeed_op_list = load_yaml(_supported_api_list_path[0]).get(Const.PT_API_TYPE_MINDSPEED) + mindspeed_op_file_list = [op.split(Const.SEP)[0] + Const.PY_SUFFIX for op in mindspeed_op_list] + dynamic_import_op(mindspeed.ops, mindspeed_op_file_list) @parameter_adapter @@ -70,7 +86,15 @@ def tensor_module_forward(module, *args, **kwargs): def dist_module_forward(module, *args, **kwargs): handle = module.api_func(*args, **kwargs) - if kwargs.get("async_op") or module.api_name in ["isend", "irecv"]: + try: + bound = inspect.signature(module.api_func).bind(*args, **kwargs) + bound.apply_defaults() + use_asyn_op_flag = bound.arguments.get("asyn_op", False) + except Exception as e: + use_asyn_op_flag = False + logger.warning(f"fail to get dist api's func signature because {e}, no wait") + + if use_asyn_op_flag or module.api_name in ["isend", "irecv"]: if handle and hasattr(handle, 'wait'): handle.wait() if module.api_name == "batch_isend_irecv": diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py index dccf9c7a9221990eb5ec3829544368ede1297b2c..f8c1d2d6f557f2f90ff348db6f230364db73161c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py @@ -21,9 +21,7 @@ import torch import torch.nn as nn import torch.utils.hooks as full_hooks -from msprobe.pytorch.common.utils import is_float8_tensor - -torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' +from msprobe.pytorch.common.utils import is_float8_tensor, register_forward_pre_hook, register_forward_hook class HOOKModule(nn.Module): @@ -43,13 +41,9 @@ class HOOKModule(nn.Module): prefix = self.prefix_api_name if hasattr(self, "prefix_api_name") else "" if callable(hook_build_func): - forward_pre_hook, forward_hook, backward_hook, _ = hook_build_func(prefix) - if torch_version_above_or_equal_2: - self.register_forward_pre_hook(forward_pre_hook, with_kwargs=True) - self.register_forward_hook(forward_hook, with_kwargs=True) - else: - self.register_forward_pre_hook(forward_pre_hook) - self.register_forward_hook(forward_hook) + forward_pre_hook, forward_hook, backward_hook = hook_build_func(prefix) + register_forward_pre_hook(self, forward_pre_hook) + register_forward_hook(self, forward_hook) self.register_backward_hook(backward_hook) def __call__(self, *args, **kwargs): @@ -79,13 +73,7 @@ class HOOKModule(nn.Module): if len(self._backward_hooks) > 0: full_backward_hooks, non_full_backward_hooks = self._get_backward_hooks() for hook in self._forward_pre_hooks.values(): - result_args, result_kwargs = hook(self, args, kwargs) - if result_args is not None: - if not isinstance(result_args, tuple): - result_args = (result_args,) - args = result_args - if result_kwargs is not None: - kwargs = result_kwargs + hook(self, args, kwargs) bw_hook = None if len(full_backward_hooks) > 0: bw_hook = full_hooks.BackwardHook(self, full_backward_hooks) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/jit_script_wrapper.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/jit_script_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..ea2ee39ae79544b5a699800cb1e7dc9e0fc9066b --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/jit_script_wrapper.py @@ -0,0 +1,33 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from msprobe.pytorch.hook_module.api_register import get_api_register + + +def wrap_jit_script_func(): + def patched_script(*args, **kwargs): + all_api_registered = api_register.all_api_registered + if all_api_registered: + api_register.restore_all_api() + result = original_script(*args, **kwargs) + if all_api_registered: + api_register.register_all_api() + return result + + original_script = torch.jit.script + api_register = get_api_register() + torch.jit.script = patched_script diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml index 15603758f52475d9d401de37e5113a4541b67572..f2d5d22ade2c52057b969a93b73e0897e5d64ae3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml @@ -1258,6 +1258,7 @@ torch_npu: - npu_scatter_nd_update_ - npu_scatter_nd_update - npu_prefetch + - npu_dynamic_block_quant aten: - signbit @@ -2009,4 +2010,23 @@ distributed: npu_distributed: - isend - - irecv \ No newline at end of file + - irecv + +mindspeed: + - dropout_add_layer_norm.npu_dropout_add_layer_norm + - npu_rotary_position_embedding.npu_rotary_position_embedding + - fusion_attention_v2.npu_fusion_attention + - npu_mm_all_reduce_add_rms_norm.npu_mm_all_reduce_add_rms_norm + - npu_mm_all_reduce_add_rms_norm_.npu_mm_all_reduce_add_rms_norm_ + - gmm.npu_gmm + - gmm.npu_gmm_v2 + - npu_grouped_mat_mul_all_reduce.npu_grouped_mat_mul_all_reduce + - ffn.npu_ffn + - npu_moe_token_permute.npu_moe_token_permute + - npu_moe_token_unpermute.npu_moe_token_unpermute + - npu_ring_attention_update.npu_ring_attention_update + - npu_matmul_add.npu_matmul_add_fp32 + - npu_groupmatmul_add.npu_groupmatmul_add_fp32 + - quant_gmm.npu_quant_gmm + - quant_gmm.npu_quant_gmm_v2 + - npu_apply_fused_ema_adamw.npu_apply_fused_ema_adamw \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py index 41869403a547fc526ec422ecbb123af18ff81a39..68e434d0ad151fc70d2a7bbb333b195d4bbe0e2f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +14,11 @@ # limitations under the License. import os -from msprobe.core.common.file_utils import load_yaml +import importlib +import inspect + +from msprobe.core.common.file_utils import load_yaml, check_link +from msprobe.core.common.log import logger def get_ops(): @@ -26,3 +30,25 @@ def get_ops(): wrap_torch = ops.get('torch') wrap_npu_ops = ops.get('torch_npu') return set(wrap_functional) | set(wrap_tensor) | set(wrap_torch) | set(wrap_npu_ops) + + +def dynamic_import_op(package, white_list): + package_name = package.__name__ + ops = {} + ops_dir, _ = os.path.split(package.__file__) + check_link(ops_dir) + for file_name in os.listdir(ops_dir): + if file_name in white_list: + sub_module_name = file_name[:-3] + module_name = f"{package_name}.{sub_module_name}" + try: + module = importlib.import_module(module_name) + except Exception as e: + logger.warning(f"import {module_name} failed!") + continue + + func_members = inspect.getmembers(module, inspect.isfunction) + for func_member in func_members: + func_name, func = func_member[0], func_member[1] + ops[f"{sub_module_name}.{func_name}"] = func + return ops diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/anomaly_analyse.py b/debug/accuracy_tools/msprobe/pytorch/monitor/anomaly_analyse.py index 9a0b71e8a5791bc216c82737d1d4f4a482abceb9..f1bdaa35ef7dea1471c5d54fbefa513c410126b3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/anomaly_analyse.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/anomaly_analyse.py @@ -21,7 +21,7 @@ import heapq from msprobe.pytorch.common.log import logger from msprobe.core.common.const import MonitorConst -from msprobe.core.common.file_utils import check_path_before_create, save_json, create_directory, remove_path, \ +from msprobe.core.common.file_utils import save_json, create_directory, remove_path, \ check_file_or_directory_path, load_json from msprobe.pytorch.monitor.anomaly_detect import GradAnomalyData @@ -46,12 +46,7 @@ class AnomalyDataWriter: def init_detected_json(self): """初始化落盘文件""" - check_path_before_create(self.dump_path) - if not os.path.exists(self.dump_path): - create_directory(self.dump_path) - - if not os.path.exists(self.dump_rank_dir): - create_directory(self.dump_rank_dir) + create_directory(self.dump_rank_dir) if os.path.exists(self.json_path): check_file_or_directory_path(self.json_path, isdir=False) diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/anomaly_detect.py b/debug/accuracy_tools/msprobe/pytorch/monitor/anomaly_detect.py index 63f20b1928c80e1e29d7cb8224f267c246fcaa8b..13ecdd8b500aa00d7459a1d5a7b2959f1e3a509b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/anomaly_detect.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/anomaly_detect.py @@ -14,6 +14,7 @@ # limitations under the License. import itertools import os +import math import statistics as st import sys from abc import ABC @@ -33,7 +34,7 @@ from msprobe.pytorch.common.log import logger class ScanRule(ABC): name = "ScanRule" - def apply(self, history, cur): + def apply(self, cur, history=None): raise NotImplementedError("abstract method apply is not implemented") @@ -43,14 +44,25 @@ class AnomalyTurbulence(ScanRule): def __init__(self, threshold) -> None: self.threshold = threshold - def apply(self, history, cur): + def apply(self, cur, history=None): + """ + :param cur: float, current metric value + :param history: float, history weighted average + :return: bool, whether the current value deviates from the historical average value of current metric + """ baseline = st.mean(history) if isinstance(history, list) else history + up_bound = baseline * (1 + self.threshold) + return abs(cur) > up_bound - up_bound = baseline + baseline * self.threshold - if baseline > 0: - return cur > up_bound - else: - return cur < up_bound + +class AnomalyNan(ScanRule): + name = "AnomalyNan" + + def __init__(self, threshold=None) -> None: + self.threshold = threshold + + def apply(self, cur, history=None): + return math.isnan(cur) or (self.threshold is not None and abs(cur) > self.threshold) class AnomalyScanner: @@ -69,7 +81,7 @@ class AnomalyScanner: rule_args = spec.get("args") # 检查必要的键是否存在 - if rule_cls_name is None or rule_args is None: + if rule_cls_name is None or (rule_cls_name == "AnomalyTurbulence" and rule_args is None): logger.warning(f"Spec is missing required keys: {spec}") continue @@ -81,7 +93,7 @@ class AnomalyScanner: continue try: - rule_instance = rule_cls(**rule_args) + rule_instance = rule_cls(**rule_args) if rule_args is not None else rule_cls() alert_rules.append(rule_instance) except Exception as e: logger.error(f"Error creating instance of rule '{rule_cls_name}': {e}") @@ -93,7 +105,7 @@ class AnomalyScanner: def scan(scan_rules: List[ScanRule], history, cur): anomaly = False for rule in scan_rules: - anomaly = rule.apply(history, cur) + anomaly = rule.apply(cur, history=history) if anomaly: return anomaly, rule.name return anomaly, None @@ -162,7 +174,7 @@ class TrainStage: FORWARD_KEY = [MonitorConst.ACTV] -BACKWARD_KEY = [MonitorConst.ACTVGRAD, MonitorConst.PRE_GRAD, +BACKWARD_KEY = [MonitorConst.ACTVGRAD, MonitorConst.PRE_GRAD, MonitorConst.POST_GRAD, MonitorConst.ACC_GRAD] OPTIMIZER_KEY = [MonitorConst.EXP_AVG, MonitorConst.EXP_AVG_SQ] TRAIN_STAGE = { @@ -253,6 +265,41 @@ class BaseWriterWithAD: self.anomaly_factory = writer_input.anomaly_factory self.anomalies = [] self.ndigits = writer_input.ndigits + self.beta = 0.99 + + @staticmethod + def stack_tensors(tensor_list): + """ + Torch not support stack cpu and xpu tensors. Group the tensors into cpu_group and xpu_group, + stack them separately, migrate xpu_group to cpu, and then restore in the order of input. + + :param tensor_list: [tensor(-1.6165), tensor(-1.0985), tensor(-1.7777), tensor(-1.8408, device='npu:0')] + :return: result: list of float + """ + cpu_tensors = [] + xpu_tensors = [] + + for tensor in tensor_list: + if isinstance(tensor, torch.Tensor) and tensor.device.type != 'cpu': + # 将device上的tensor先stack后to cpu + xpu_tensors.append(tensor) + else: + cpu_tensors.append(tensor) + + xpu_stack = torch.stack(xpu_tensors).cpu() if xpu_tensors else torch.tensor([]) + + # 按照输入的顺序恢复 + result = [] + cpu_tensors_idx, xpu_tensors_idx = 0, 0 + for tensor in tensor_list: + if isinstance(tensor, torch.Tensor) and tensor.device.type != 'cpu': + result.append(xpu_stack[xpu_tensors_idx]) + xpu_tensors_idx += 1 + else: + result.append(cpu_tensors[cpu_tensors_idx]) + cpu_tensors_idx += 1 + + return result def get_anomalies(self): """返回已检测到的异常列表 @@ -271,12 +318,17 @@ class BaseWriterWithAD: Returns: None """ - detected = False - if self.ad_rules: - avg = self._update_tag2scalars(tag, scalar_value) - detected, rule_name = self._ad(scalar_value, history=avg) + if not self.ad_rules or tag[-1] in ["shape", "dtype"]: + return + if isinstance(scalar_value, torch.Tensor): + scalar_value = scalar_value.item() + avg = self._update_tag2scalars(tag, scalar_value) + detected, rule_name = self._ad(scalar_value, history=avg) if detected: - exception_message = f"Rule {rule_name} reports anomaly signal in {tag} at step {global_step}." + if rule_name == AnomalyTurbulence.name and tag[-1] not in ["norm", "mean"]: + return + exception_message = (f"Rule {rule_name} reports anomaly signal in {tag} at step {global_step}, " + f"current value {scalar_value}, history mean {avg}.") logger.info(f"{BCOLORS.WARNING}> {exception_message}{BCOLORS.ENDC}") # append to self.anomalies for dump if self.anomaly_factory: @@ -291,15 +343,15 @@ class BaseWriterWithAD: tensors.extend(op2tensor.values()) if not tensors: return - + n_slices = len(tensors) // MonitorConst.SLICE_SIZE with torch.no_grad(): for i in range(n_slices + 1): begin = i * MonitorConst.SLICE_SIZE - end = (i+1) * MonitorConst.SLICE_SIZE + end = (i + 1) * MonitorConst.SLICE_SIZE if begin == len(tensors): continue - metric_list = torch.stack(tensors[begin:end]).cpu() + metric_list = self.stack_tensors(tensors[begin:end]) for tag, metric in zip(tags[begin:end], metric_list): self.add_scalar(tag, metric, step) @@ -319,11 +371,11 @@ class BaseWriterWithAD: Returns: float: The average value before update. """ + abs_scalar_value = abs(scalar_value) if tag not in self.tag2scalars: - self.tag2scalars[tag] = {'avg': scalar_value, 'count': 0} + self.tag2scalars[tag] = {'avg': abs_scalar_value, 'count': 0} avg = self.tag2scalars[tag]['avg'] - new_avg = (avg * self.tag2scalars[tag]['count'] + scalar_value) / (self.tag2scalars[tag]['count'] + 1) - self.tag2scalars[tag]['avg'] = new_avg + self.tag2scalars[tag]['avg'] = self.beta * avg + (1 - self.beta) * abs_scalar_value self.tag2scalars[tag]['count'] += 1 return avg @@ -376,7 +428,13 @@ class CSVWriterWithAD(BaseWriterWithAD): super().add_scalar(tag, scalar_value, global_step) name = tag[0].split('/')[0] - self.context_dict[name].append(scalar_value.item()) + if isinstance(scalar_value, torch.Tensor): + value = scalar_value.item() + elif isinstance(scalar_value, torch.Size): + value = list(scalar_value) + else: + value = scalar_value + self.context_dict[name].append(value) def write_metrics(self, ops, metric_value, step, prefix=''): super().write_metrics(ops, metric_value, step, prefix='') diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/csv2tb.py b/debug/accuracy_tools/msprobe/pytorch/monitor/csv2tb.py index 6ffd1ffabe7b113ff4e61786d4d9f0709b8b605b..467e056ef63ccade970c66ce6ffd9b5fcf9ff835 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/csv2tb.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/csv2tb.py @@ -22,13 +22,18 @@ from torch.utils.tensorboard import SummaryWriter from tqdm import tqdm from msprobe.core.common.const import MonitorConst -from msprobe.core.common.file_utils import read_csv, create_directory, remove_path +from msprobe.core.common.file_utils import read_csv, create_directory, remove_path, recursive_chmod from msprobe.core.common.utils import is_int +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.common.log import logger from msprobe.pytorch.monitor.utils import get_target_output_dir -all_data_type_list = ["actv", "actv_grad", "exp_avg", "exp_avg_sq", "grad_unreduced", "grad_reduced", "param"] +all_data_type_list = [ + "actv", "actv_grad", "exp_avg", "exp_avg_sq", + "grad_unreduced", "grad_reduced", "param_origin", "param_updated" +] CSV_FILE_SUFFIX = r"_\d+-\d+\.csv" +MAX_PROCESS_NUM = 128 def parse_step_line(line, ops): @@ -74,8 +79,10 @@ def write_step(output_dirpath, parse_step_result, rank, data_type): for op, value in ops.items(): tag = f"{vpp_name}/{op}" writer.add_scalar(tag, value, step) + writer.flush() +@recursion_depth_decorator("update_dict", max_depth=50) def update_dict(dict1, dict2): for key, value in dict2.items(): if key in dict1: @@ -115,11 +122,13 @@ def csv2tb_by_step_work(target_output_dirs, output_dirpath, data_type_list): def check_process_num(process_num): if not is_int(process_num) or process_num <= 0: raise ValueError(f"process_num({process_num}) is not a positive integer") + if process_num > MAX_PROCESS_NUM: + raise ValueError(f"The maximum supported process_num is {MAX_PROCESS_NUM}, current value: {process_num}.") def check_data_type_list(data_type_list): if data_type_list is None: - logger.info(f"data_type_list is None, use defualt all_data_type_list: {all_data_type_list}") + logger.info(f"data_type_list is None, use default all_data_type_list: {all_data_type_list}") return if not isinstance(data_type_list, list): raise ValueError(f"data_type_list({data_type_list}) is not a list") @@ -161,4 +170,5 @@ def csv2tensorboard_by_step( p.start() for p in processes: p.join() + recursive_chmod(output_dirpath) logger.info(f"output has been saved to: {output_dirpath}") diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/distributed/wrap_distributed.py b/debug/accuracy_tools/msprobe/pytorch/monitor/distributed/wrap_distributed.py index 20ef3757d4ad45cc2bb90769f44eef1cebe82560..c209fdba97fa9a4a153516d340892fbefbf0284f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/distributed/wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/distributed/wrap_distributed.py @@ -142,7 +142,7 @@ def get_process_group(process_group): def stack_filter(stack): if len(stack) > MAX_STRING_LENGTH: - logger.warning(f'The character strin contains more than {MAX_STRING_LENGTH}. re match is skipped.') + logger.warning(f'The character string contains more than {MAX_STRING_LENGTH}. re match is skipped.') for pattern in StackBlackList: if re.search(pattern, stack): return False @@ -197,7 +197,7 @@ def is_target_line(codeline): stack = get_callstack() whole_stack = ';'.join(stack) if len(whole_stack) > MAX_STRING_LENGTH: - logger.warning(f'The character strin contains more than {MAX_STRING_LENGTH}. re match is skipped.') + logger.warning(f'The character string contains more than {MAX_STRING_LENGTH}. re match is skipped.') for pattern in codeline: if re.search(pattern, whole_stack): return True diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py b/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py index 2db2a9712566e41406ff9e82d1e4e8cff4b32ef9..3419ba17cae303133e8b54f28db3f809a2a710a1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py @@ -22,13 +22,14 @@ from functools import partial import pytz import torch import torch.distributed as dist +import pandas as pd from torch.utils.hooks import BackwardHook from msprobe.core.common.const import MonitorConst, Const from msprobe.core.common.file_utils import load_json, save_json -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import is_recomputation +from msprobe.pytorch.common.utils import is_recomputation, is_float8_tensor from msprobe.pytorch.monitor.anomaly_analyse import AnomalyDataWriter from msprobe.pytorch.monitor.anomaly_detect import AnomalyScanner, SummaryWriterWithAD, AnomalyDataFactory, \ CSVWriterWithAD, BaseWriterWithAD, WriterInput @@ -37,11 +38,12 @@ from msprobe.pytorch.monitor.distributed.wrap_distributed import api_register, c from msprobe.pytorch.monitor.features import get_sign_matches from msprobe.pytorch.monitor.module_metric import get_metrics, get_summary_writer_tag_name, \ TensorMetrics, squash_param_name -from msprobe.pytorch.monitor.module_spec_verifier import validate_config_spec from msprobe.pytorch.monitor.optimizer_collect import OptimizerMonFactory from msprobe.pytorch.monitor.utils import get_param_struct, validate_config, validate_ops, \ - get_output_base_dir, get_target_output_dir + get_output_base_dir, get_target_output_dir, chmod_tensorboard_dir, validate_set_monitor from msprobe.pytorch.monitor.visualizer import HeatmapVisualizer +from msprobe.core.common.file_utils import write_df_to_csv +from msprobe.core.common.utils import analyze_api_call_stack torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' if not torch_version_above_or_equal_2: @@ -72,36 +74,7 @@ class ModuleHookContext: self.actvgrad = [] self.module_name = module_name self.struct = {} - self.format_by_arg = {} - self.verified = False - self.focused_in_col = 0 - self.focused_out_col = 0 - - def set_format_by_arg(self, key_name: str, target_config: dict): - """ 按照监控对象配置format_by_arg - 1) module_name 在 target 中配置监控对象 - 2) module_name 未在 targets 中配置，且 all_xy 全量监控 - 3) module_name 未在 targets 中配置，且 all_xy 未全量监控 - - :param key_name: str, one of [input, output, input_grad, output_grad] - :param target_config: target obj in config json. - :return: - """ - cared = target_config.get(self.module_name, self.struct) - if key_name in cared: - target_module_config = cared[key_name] - if isinstance(target_module_config, dict): - # current cared is self.struct, monitor all data for module_name - self.format_by_arg[key_name] = target_module_config.get('config') - elif isinstance(target_module_config, str): - # current cared is target_config[self.module_name] - self.format_by_arg[key_name] = target_module_config - else: - logger.warning_on_rank_0(f"target module config error, result maybe empty." - f"module_name: {self.module_name}, key_name: {key_name}") - self.format_by_arg[key_name] = None - else: - self.format_by_arg[key_name] = self.struct.get(key_name).get('config') + self.stack = "" def reset(self): self.actv.clear() @@ -185,8 +158,8 @@ class TrainerMon: self.params_have_main_grad = params_have_main_grad self.update_heatmap_visualizer = defaultdict(HeatmapVisualizer) self.ratio_heatmap_visualizer = defaultdict(HeatmapVisualizer) - self.origin_step_func = None self.origin_start_grad_sync = None + self.fsdp_post_backward_hook = None self.config_timestamp = 0 # 后面有校验时间戳, 首次监控无需为了更新config文件时间戳而去改, 可通过dynamic_on开关直接打开 self.config = load_json(config_file_path) validate_config(self.config) @@ -221,8 +194,8 @@ class TrainerMon: self.dp_group = None self.tp_group = None self.enable_megatron = False + self.fsdp_wrapped_module = False self.micro_batch_number = 1 - self.optimizer_class = None self.optimizer_mon = None self.optimizer_trans = None @@ -234,7 +207,6 @@ class TrainerMon: self.grad_context = GradContext() self.handles = defaultdict(list) self.param2name = defaultdict(str) - self.name2index = defaultdict() self.name2indices = defaultdict() self.name2param = {} self.duplicate_param = {} @@ -247,6 +219,8 @@ class TrainerMon: self.optimizer_hooked = False self.param_registered = False self.struct_printed = False + self.pre_step_hooks = [] + self.post_step_hooks = [] # 动静态区分 self.dynamic_enable = os.getenv("DYNAMIC_MONITOR", 'False').lower() == 'true' @@ -317,6 +291,7 @@ class TrainerMon: self.param_distribution = self.config.get("param_distribution", False) self.mg_direction = self.config.get('mg_direction', False) self.cc_distribution = self.config.get("cc_distribution", {}) + self.stack_info = self.config.get('stack_info', False) if not self.cc_distribution.get('enable', False): self.cc_log_only = False @@ -325,8 +300,6 @@ class TrainerMon: self.cc_log_only = self.cc_distribution.get('cc_log_only', False) self.cc_logged_stack = defaultdict(set) self.cc_pre_hook = self.cc_distribution.get('cc_pre_hook', False) - self.handles['cc'] = api_register.initialize_hook(*create_hooks(context=self.cc_context, monitor=self)) - api_register.redirect_api() self.common_info() @@ -339,11 +312,11 @@ class TrainerMon: # 初始化writer, 创建输出目录 if self.format not in FORMAT_MAPPING: - logger.error(f"Unsupported format: {self.format}, use default format: {MonitorConst.CSV}") + logger.warning(f"Unsupported format: {self.format}, use default format: {MonitorConst.CSV}") self.format = MonitorConst.CSV if self.ur_distribution and self.format != 'tensorboard': - logger.error("can only set ur_distribution when format is 'tensorboard', cancel ur_distribution") + logger.warning("can only set ur_distribution when format is 'tensorboard', cancel ur_distribution") self.ur_distribution = False writer = FORMAT_MAPPING[self.format] @@ -406,13 +379,14 @@ class TrainerMon: start_iteration=0 ): """External interface""" + grad_acc_steps, start_iteration = validate_set_monitor(grad_acc_steps, start_iteration) global start_step start_step = start_iteration logger.info(f'grad acc steps {grad_acc_steps}') self.micro_batch_number = grad_acc_steps self.dp_group = dp_group self.tp_group = tp_group - self.optimizer_mon, self.optimizer_class = OptimizerMonFactory.create_optimizer_mon(optimizer) + self.optimizer_mon = OptimizerMonFactory.create_optimizer_mon(optimizer) self.hook_step_final(optimizer) if not isinstance(model, list): model = [model] @@ -428,6 +402,9 @@ class TrainerMon: self.hook_optimizer(optimizer) self._patch_grad_sync() self.hook_modules() + if self.cc_distribution.get('enable', False): + self.handles['cc'] = api_register.initialize_hook(*create_hooks(context=self.cc_context, monitor=self)) + api_register.redirect_api() self.monitoring = True def adhoc_check(self, target_tensor: torch.tensor, module_name: str, tensor_name: str, rank_list, ops_list): @@ -438,25 +415,48 @@ class TrainerMon: return self.tensor_metrics.stat_insert(target_tensor, ops_list, module_name, tensor_name, rank) - def build_tbtag_tensor_map(self, module_name, tag, tensor): - key = get_summary_writer_tag_name(module_name, tag, self.rank) - self._register_param_call_id("_hook_module", key) - return {key: tensor} + def build_tbtag_tensor_map(self, module_name, suffix, tag, tensor): + """ + :param module_name: str of module name + :param suffix: + :param tag: + :param tensor: torch.tensor or tuple/list of torch.tensor + :return: tensor_map + """ + tensor_map = {} + if isinstance(tensor, torch.Tensor): + tensor = [tensor] + if isinstance(tensor, tuple) or isinstance(tensor, list): + if len(tensor) == 1: + key = get_summary_writer_tag_name(module_name + suffix, tag, self.rank) + self.register_param_call_id("_hook_module", key) + tensor_map[key] = tensor[0] + else: + for i, tensor_i in enumerate(tensor): + key = get_summary_writer_tag_name(module_name + f"_{i}" + suffix, tag, self.rank) + self.register_param_call_id("_hook_module", key) + tensor_map[key] = tensor_i + return tensor_map def generate_param_map(self, tag, param_tensor): metrics = {} for name in self.param2name.values(): key = get_summary_writer_tag_name(name, tag, self.rank) - self._register_param_call_id("optimizer_pre_step_hook", key) + self.register_param_call_id("optimizer_pre_step_hook", key) if name not in param_tensor or param_tensor[name] is None: continue metrics[key] = param_tensor[name] return metrics - def generate_param_metrics(self, opt_context): + def generate_param_metrics(self, opt_context, stage=MonitorConst.PRE_PARAM): if not self.param_distribution: return - get_metrics(self.ops, self.name2param, self.eps, opt_context.param_metric) + tag2param = { + self.name2tag.get(name, {}).get(stage): param + for name, param in self.name2param.items() + if param.numel() != 0 + } + get_metrics(self.ops, tag2param, self.eps, opt_context.param_metric) def generate_mv_metrics(self, opt_context): if not self.mv_distribution: @@ -468,28 +468,20 @@ class TrainerMon: get_metrics(self.ops, m_tag_tensor_map, self.eps, opt_context.exp_avg_metric) get_metrics(self.ops, v_tag_tensor_map, self.eps, opt_context.exp_avg_sq_metric) - def generate_wgrad_metrics(self): + def generate_wgrad_metrics(self, post_grad_dict): if not self.wg_distribution: return {}, {} if self.weight_hooked: get_metrics(self.ops, self.grad_context.acc, self.eps, self.grad_context.acc_metric) - grad_dict = {} - for param, name in self.param2name.items(): - if self.duplicate_param.get(name, False): - continue - grad = param.main_grad if self.params_have_main_grad else param.grad - if grad is None: - logger.warning(f"grad is None: {name}, maybe something wrong happened.") - continue - tag = self.name2tag.get(name, {}).get(MonitorConst.POST_GRAD) - self._register_param_call_id("hook_optimizer", tag) - grad_dict[tag] = grad - - get_metrics(self.ops, grad_dict, self.eps, self.grad_context.post) - unreduced_grad = self.grad_context.acc_metric if self.weight_hooked else self.grad_context.pre - return self.grad_context.post, unreduced_grad + get_metrics(self.ops, post_grad_dict, self.eps, self.grad_context.post) + reduced_grad = self.grad_context.post + if self.enable_megatron or self.fsdp_wrapped_module: + unreduced_grad = self.grad_context.pre + else: + unreduced_grad = self.grad_context.acc_metric + return reduced_grad, unreduced_grad def generate_xy_metrics(self): actv = {} @@ -515,6 +507,17 @@ class TrainerMon: def write_adhoc_check(self, step): self.tensor_metrics.flush(self.summary_writer) + def write_stack_info(self): + stack_data = [] + header = ["module_name", "stack_info"] + stack_data.append(header) + for _, fwd_context in self.module_fwd_hook_context_by_module.items(): + stack_data.append([fwd_context.module_name, fwd_context.stack]) + filepath = os.path.join(self.tensorboard_dir, f'stack_info.csv') + if not os.path.exists(filepath): + data_frame = pd.DataFrame(columns=stack_data) + write_df_to_csv(data_frame, filepath) + def write_xy_tb(self, step): if not self.xy_distribution: return @@ -529,7 +532,10 @@ class TrainerMon: def write_param_tb(self, opt_context): if not self.param_distribution: return - self.summary_writer.write_metrics(self.ops, opt_context.param_metric, opt_context.step, MonitorConst.PARAM) + param_metrics = {k: v for k, v in opt_context.param_metric.items() if MonitorConst.PRE_PARAM in k} + updated_param_metrics = {k: v for k, v in opt_context.param_metric.items() if MonitorConst.POST_PARAM in k} + self.summary_writer.write_metrics(self.ops, param_metrics, opt_context.step, MonitorConst.PRE_PARAM) + self.summary_writer.write_metrics(self.ops, updated_param_metrics, opt_context.step, MonitorConst.POST_PARAM) def write_mv_tb(self, opt_context): if not self.mv_distribution: @@ -543,7 +549,7 @@ class TrainerMon: if not self.wg_distribution: return - if self.enable_megatron: + if self.enable_megatron or self.fsdp_wrapped_module: self.summary_writer.write_metrics(self.ops, self.grad_context.pre, step, 'grad_unreduced') else: self.summary_writer.write_metrics(self.ops, self.grad_context.acc_metric, step, 'grad_unreduced') @@ -568,21 +574,23 @@ class TrainerMon: # skip generate metrics if context.step < self.start_step or (context.step - self.start_step) % self.step_interval != 0: return - if MonitorConst.DEEPSPEED_ZERO_OPT_FILTER in self.optimizer_class: # use deepspeed with zero1/2/3 - if not self.name2indices: - self.name2indices = self.optimizer_mon.get_param_index(self.param2name, self.name2index, optimizer) - mv_result = self.optimizer_mon.fetch_mv(self, optimizer, self.param2name, self.name2indices) - self.param2name = mv_result.grad - else: - mv_result = self.optimizer_mon.fetch_mv(self, optimizer, self.param2name) - context.param_exp_avg = mv_result.exp_avg - context.param_exp_avg_sq = mv_result.exp_avg_sq - context.param_adam_update = mv_result.update - context.param_adam_ratio = mv_result.ratio - self.generate_wgrad_metrics() + grad_dict = {} + if self.wg_distribution: + grad_dict = self.optimizer_mon.fetch_grad(self, self.param2name) + + mv_result = None + if self.mv_distribution or self.ur_distribution or self.mg_direction: + mv_result = self.optimizer_mon.fetch_mv(self, self.param2name) + if mv_result: + context.param_exp_avg = mv_result.exp_avg + context.param_exp_avg_sq = mv_result.exp_avg_sq + context.param_adam_update = mv_result.update + context.param_adam_ratio = mv_result.ratio + + self.generate_wgrad_metrics(grad_dict) self.generate_mv_metrics(context) - self.generate_param_metrics(context) + self.generate_param_metrics(context, MonitorConst.PRE_PARAM) tbtag_tensor_map = {} if self.mg_direction: @@ -610,17 +618,15 @@ class TrainerMon: context.metric_dict = metric_dict return - def patch_step(func, optimizer): - def wrapper(*args, **kwargs): - optimizer_pre_step_hook(optimizer, args, kwargs) - out = func(*args, **kwargs) - return out - return wrapper + def optimizer_post_step_hook(optimizer, args, kwargs): + context = self.optimizer_context[optimizer] + self.generate_param_metrics(context, MonitorConst.POST_PARAM) if self.optimizer_hooked: return - optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) + self.pre_step_hooks.append(optimizer_pre_step_hook) + self.post_step_hooks.append(optimizer_post_step_hook) self.optimizer_hooked = True return @@ -650,6 +656,7 @@ class TrainerMon: validate_config(config) self.config = config self.set_config() + self.start_step = context.step # 动态启停时不受原start_step影响，永远从下一步开始 logger.warning(f"config is updated at step{context.step - 1}, " f"will start new hook at step{context.step}.") except Exception as e: @@ -679,6 +686,12 @@ class TrainerMon: self.write_mv_tb(context) self.write_param_tb(context) self.write_adhoc_check(context.step) + if self.stack_info: + self.write_stack_info() + self.stack_info = False + for handle in self.handles["stack"]: + handle.remove() + self.handles["stack"].clear() if self.ur_distribution: for param_name, _ in context.param_adam_update.items(): @@ -697,6 +710,9 @@ class TrainerMon: if self.anomaly_data_factory: self.anomaly_data_writer.write_detected_json(self.summary_writer.get_anomalies()) self.summary_writer.clear_anomalies() + + if self.format == MonitorConst.TENSORBOARD: + chmod_tensorboard_dir(self.tensorboard_dir) self.call_id = 0 self.param_name_call_id.clear() @@ -708,13 +724,16 @@ class TrainerMon: def patch_step(func, optimizer): def wrapper(*args, **kwargs): + for hook in self.pre_step_hooks: + hook(optimizer, args, kwargs) out = func(*args, **kwargs) + for hook in self.post_step_hooks: + hook(optimizer, args, kwargs) step_final_hook(optimizer, args, kwargs) return out return wrapper optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) - self.origin_step_func = optimizer.__class__.step return def hook_modules(self): @@ -740,7 +759,7 @@ class TrainerMon: def clone_if_tensor(args): if isinstance(args, tuple): return tuple([clone_if_tensor(arg) for arg in args]) - elif isinstance(args, torch.Tensor): + elif isinstance(args, torch.Tensor) and not is_float8_tensor(args): return args.clone() else: return args @@ -758,6 +777,16 @@ class TrainerMon: BackwardHook.setup_output_hook = wrap_hook_setup(BackwardHook.setup_output_hook) return + def register_param_call_id(self, hook_name: str, key: str): + """ + :param hook_name: + :param key: str, '0:relu_0/output_grad' + :return: + """ + logger.debug(f"{hook_name} {key}: {self.call_id}") + self.param_name_call_id[key] = self.call_id + self.call_id += 1 + def _remove_all_hooks(self, optimizer): # 清空hook handle for handle in self.handles['xy']: @@ -783,14 +812,18 @@ class TrainerMon: logger.info("remove _ParamAndGradBucketGroup start_grad_sync") except ImportError: pass - else: # not megatron + elif self.fsdp_post_backward_hook: # fsdp + torch.distributed.fsdp._runtime_utils._post_backward_hook = self.fsdp_post_backward_hook + logger.info("remove patch_post_backward_hook in fsdp.") + else: # not megatron and not fsdp for handle in self.handles['wgrads']: handle.remove() self.handles['wgrads'].clear() self.weight_hooked = False if self.optimizer_hooked: - optimizer.__class__.step = self.origin_step_func + self.pre_step_hooks.clear() + self.post_step_hooks.clear() for _, context in self.optimizer_context.items(): context.reset() @@ -799,12 +832,12 @@ class TrainerMon: for handle in self.handles['cc']: handle.remove() self.handles['cc'].clear() + api_register.restore_api() for _, context in self.cc_context.items(): context.reset() # 清空节点缓存 self.param2name.clear() - self.name2index.clear() self.name2indices.clear() self.name2param.clear() self.duplicate_param.clear() @@ -864,27 +897,33 @@ class TrainerMon: return False def _register_chunk(self, model_chunk, prefix): - index = 0 for (param_name, param) in model_chunk.named_parameters(): if not param.requires_grad: continue + if not self.fsdp_wrapped_module and param_name.startswith("_fsdp_wrapped_module"): + self.fsdp_wrapped_module = True if self._is_target_param(param_name, param, prefix): name = prefix + squash_param_name(param_name, self.squash_name) if name in self.param2name.values(): name = prefix + param_name self.param2name[param] = name self.name2param[name] = param - self.name2index[name] = index if self.tp_group and not param_is_not_tensor_parallel_duplicate(param, self.tp_group): self.duplicate_param[name] = True if self.dp_group and param_is_data_parallel_duplicate(self.dp_group): self.duplicate_param[name] = True + + keywords = [ + MonitorConst.PRE_GRAD, + MonitorConst.POST_GRAD, + MonitorConst.PRE_PARAM, + MonitorConst.POST_PARAM + ] self.name2tag[name] = { - MonitorConst.PRE_GRAD: get_summary_writer_tag_name(name, MonitorConst.PRE_GRAD, self.rank), - MonitorConst.POST_GRAD: get_summary_writer_tag_name(name, MonitorConst.POST_GRAD, self.rank) + k: get_summary_writer_tag_name(name, k, self.rank) + for k in keywords } - index += 1 def _register_param_name(self): for vpp_stage, model_chunk in enumerate(self.model): @@ -907,11 +946,17 @@ class TrainerMon: # nothing to hook return 0 - def fwd_hook_fun(module, module_input, module_output, name): + def fwd_hook_fun(module, args, kwargs, module_output, name): if not module.training or is_recomputation(): # 1 only monitor training stage. # 2 when open recompute, skip recomputed forward stage. return + + module_input = [tensor for tensor in args if torch.is_tensor(tensor)] + if kwargs: + kwargs_tensors = [tensor for tensor in kwargs.values() if torch.is_tensor(tensor)] + module_input.extend(kwargs_tensors) + if module not in self.module_fwd_hook_context_by_module: self.module_fwd_hook_context_by_module[module] = ModuleHookContext(name) context: ModuleHookContext = self.module_fwd_hook_context_by_module[module] @@ -920,34 +965,20 @@ class TrainerMon: Const.INPUT: get_param_struct(module_input), Const.OUTPUT: get_param_struct(module_output) } + if self.print_struct: self.module_struct[context.module_name].update(context.struct) return - if not context.format_by_arg: - context.set_format_by_arg(Const.INPUT, self.config['targets']) - context.set_format_by_arg(Const.OUTPUT, self.config['targets']) - if not context.format_by_arg: - return - if not context.verified: - context.focused_in_col = validate_config_spec(context.format_by_arg[Const.INPUT], - module_input, context.module_name, - Const.INPUT) - context.focused_out_col = validate_config_spec(context.format_by_arg[Const.OUTPUT], - module_output, context.module_name, - Const.OUTPUT) - context.verified = True - # expect output be tensor type + tbtag_tensor_map = {} - cared_input = module_input if context.focused_in_col is None else module_input[context.focused_in_col] tbtag_tensor_map.update( self.build_tbtag_tensor_map( - f'{context.module_name}.{Const.INPUT}{MonitorConst.NAME_SEP}{context.micro_step}', - MonitorConst.ACTV, cared_input)) - cared_output = module_output if context.focused_out_col is None else module_output[context.focused_out_col] + f'{context.module_name}.{Const.INPUT}', f'{MonitorConst.NAME_SEP}{context.micro_step}', + MonitorConst.ACTV, module_input)) tbtag_tensor_map.update( self.build_tbtag_tensor_map( - f'{context.module_name}.{Const.OUTPUT}{MonitorConst.NAME_SEP}{context.micro_step}', - MonitorConst.ACTV, cared_output)) + f'{context.module_name}.{Const.OUTPUT}', f'{MonitorConst.NAME_SEP}{context.micro_step}', + MonitorConst.ACTV, module_output)) get_metrics(self.ops, tbtag_tensor_map, self.eps, context.actv) context.micro_step += 1 @@ -965,31 +996,17 @@ class TrainerMon: if self.print_struct: self.module_struct[context.module_name].update(context.struct) return - if not context.format_by_arg: - context.set_format_by_arg(MonitorConst.INPUT_GRAD, self.config['targets']) - context.set_format_by_arg(MonitorConst.OUTPUT_GRAD, self.config['targets']) - if not context.format_by_arg: - return - if not context.verified: - context.focused_in_col = validate_config_spec( - context.format_by_arg[MonitorConst.INPUT_GRAD], - input_grad, context.module_name, MonitorConst.INPUT_GRAD) - context.focused_out_col = validate_config_spec( - context.format_by_arg[MonitorConst.OUTPUT_GRAD], - output_grad, context.module_name, MonitorConst.OUTPUT_GRAD) - context.verified = True tbtag_tensor_map = {} - cared_input_grad = input_grad if context.focused_in_col is None else input_grad[context.focused_in_col] tbtag_tensor_map.update( self.build_tbtag_tensor_map( - f'{context.module_name}.{Const.INPUT}{MonitorConst.NAME_SEP}{context.micro_step}', - MonitorConst.ACTV, cared_input_grad)) - cared_output_grad = output_grad if context.focused_out_col is None else output_grad[context.focused_out_col] + f'{context.module_name}.{Const.INPUT}', f'{MonitorConst.NAME_SEP}{context.micro_step}', + MonitorConst.ACTVGRAD, input_grad)) + tbtag_tensor_map.update( self.build_tbtag_tensor_map( - f'{context.module_name}.{Const.OUTPUT}{MonitorConst.NAME_SEP}{context.micro_step}', - MonitorConst.ACTV, cared_output_grad)) + f'{context.module_name}.{Const.OUTPUT}', f'{MonitorConst.NAME_SEP}{context.micro_step}', + MonitorConst.ACTVGRAD, output_grad)) if context.micro_step == 0 and context.actvgrad: logger.warning(f"actvgrad context of {context.module_name} is not empty when first micro_step, " @@ -1003,17 +1020,30 @@ class TrainerMon: context.micro_step = 0 return + def stack_hook(module, args, kwargs, module_output, name): + if module not in self.module_fwd_hook_context_by_module: + self.module_fwd_hook_context_by_module[module] = ModuleHookContext(name) + context: ModuleHookContext = self.module_fwd_hook_context_by_module[module] + context.stack = analyze_api_call_stack(name) + return + if self.backward_only and self.forward_only: logger.warning('not enable backward_only and forward_only simultaneously') hooked_count = 0 - if self.xy_distribution or self.print_struct: - for module_name, submodule in module.named_modules(): - name = self._is_target_module(module_name, target_names, vpp_stage) - if not name: - continue + for module_name, submodule in module.named_modules(): + if self.stack_info: + name = vpp_stage + squash_param_name(module_name, self.squash_name) + handle = submodule.register_forward_hook(partial(stack_hook, name=name), with_kwargs=True) + self.handles['stack'].append(handle) + name = self._is_target_module(module_name, target_names, vpp_stage) + if not name: + continue + if submodule.__class__.__name__ == "FullyShardedDataParallel": + continue + if self.xy_distribution or self.print_struct: if not self.backward_only: - handle = submodule.register_forward_hook(partial(fwd_hook_fun, name=name)) + handle = submodule.register_forward_hook(partial(fwd_hook_fun, name=name), with_kwargs=True) self.handles['xy'].append(handle) if not self.forward_only and not self.has_register_backward_hook(name, submodule): handle = submodule.register_full_backward_hook(bwd_hook_fun) @@ -1042,7 +1072,7 @@ class TrainerMon: if tag is None: continue grad_dict[tag] = grad - self._register_param_call_id("sync_grad_func", tag) + self.register_param_call_id("sync_grad_func", tag) get_metrics(self.ops, grad_dict, self.eps, self.grad_context.pre) out = sync_grad_func(bucket) return out @@ -1051,6 +1081,10 @@ class TrainerMon: if not self.wg_distribution: return + if self.fsdp_wrapped_module: + # patch fsdp _runtime_utils._post_backward_hook + self._patch_fsdp_post_backward_hook() + return try: from megatron.core.distributed.param_and_grad_buffer import Bucket @@ -1069,9 +1103,44 @@ class TrainerMon: logger.info("megatron version is > core_r0.8.0 <= core_r0.9.0") except ImportError: self.enable_megatron = False | self.enable_megatron + if self.enable_megatron: + return + + # default hook weights + self._hook_weights() + + def _patch_fsdp_post_backward_hook(self): + """ + FSDP runtime 需要处理整个forward和backward计算和通信的流程，通过override nn.Module的forward，定义相应的逻辑。 + 对AccumulateGrad对象注册hook，可以在backward计算grad后立刻执行，在reduce_scatter操作前采集梯度累计后，通信聚合前的梯度。 + 每个forward阶段，fsdp对AccumulateGrad重复注册hook方法，monitor工具内注册hook无法生效， + 因此对_post_backward_hook进行patch，在backward后，reduce_scatter前采集梯度。 + """ + def patch_post_backward_hook(_post_backward_hook): + def wrapper(state, handle, *unused): + grad_dict = {} + offset = 0 + for param, name in self.param2name.items(): + limit = param.numel() + if not limit: + continue + grad = handle.flat_param.grad[offset:offset + limit] + offset += limit + tag = self.name2tag.get(name, {}).get(MonitorConst.PRE_GRAD) + if tag is None: + continue + grad_dict[tag] = grad + self.register_param_call_id("_post_backward_hook", tag) + get_metrics(self.ops, grad_dict, self.eps, self.grad_context.pre) + out = _post_backward_hook(state, handle, *unused) + return out + + return wrapper - if not self.enable_megatron: - self._hook_weights() + logger.info("Patch fsdp _post_backward_hook, collect pre_grad metrics.") + self.fsdp_post_backward_hook = torch.distributed.fsdp._runtime_utils._post_backward_hook + torch.distributed.fsdp._runtime_utils._post_backward_hook = \ + patch_post_backward_hook(torch.distributed.fsdp._runtime_utils._post_backward_hook) def _hook_weights(self): context = self.grad_context @@ -1079,13 +1148,16 @@ class TrainerMon: @torch.no_grad def param_hook(*args, context_dict, param, key, name): param.micro_step += 1 - self._register_param_call_id("param_hook", key) + self.register_param_call_id("param_hook", key) if param.micro_step == self.micro_batch_number: param.micro_step = 0 if self.params_have_main_grad: - context_dict[key] = param.main_grad.clone() + grad = param.main_grad else: - context_dict[key] = param.grad.clone() + grad = param.grad + if is_float8_tensor(grad): + grad = grad.float() + context_dict[key] = grad.clone() logger.info("hooking weights.") for param, name in self.param2name.items(): @@ -1099,13 +1171,3 @@ class TrainerMon: self.handles['wgrads'].append(handle) self.weight_hooked = True - - def _register_param_call_id(self, hook_name: str, key: str): - """ - :param hook_name: - :param key: str, '0:relu_0/output_grad' - :return: - """ - logger.debug(f"{hook_name} {key}: {self.call_id}") - self.param_name_call_id[key] = self.call_id - self.call_id += 1 diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/module_metric.py b/debug/accuracy_tools/msprobe/pytorch/monitor/module_metric.py index 87963812006413a90fd33bc70d6172a7c73c3f10..48d241c5f6129df05997f52c0957ee7976ff171e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/module_metric.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/module_metric.py @@ -16,6 +16,7 @@ import re import torch +from msprobe.pytorch.common.utils import is_float8_tensor from msprobe.pytorch.monitor.features import get_max, get_min, get_zeros, get_nans, get_norm, get_mean from msprobe.pytorch.monitor.utils import get_nan_tensor @@ -143,6 +144,20 @@ class IdentMetric(Metric): return tensor +@register_config_metric("shape") +class ShapeMetric(Metric): + @staticmethod + def get_metric_value(tensor, eps): + return tensor.shape + + +@register_config_metric("dtype") +class DtypeMetric(Metric): + @staticmethod + def get_metric_value(tensor, eps): + return tensor.dtype + + def get_metrics(ops, tag2tensor, eps, out_dict=None): """ :param ops: ["op1", "op2"] @@ -166,6 +181,8 @@ def get_metrics(ops, tag2tensor, eps, out_dict=None): # Non-tensor in/output filled with nan. out_dict[tag].update({metric_name: get_nan_tensor() for metric_name in ops}) continue + if is_float8_tensor(tensor): + tensor = tensor.float() for metric_name in ops: fun_metric = config_metric_registry.get(metric_name) out_dict[tag][metric_name] = fun_metric.get_metric(tensor, eps) diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/module_spec_verifier.py b/debug/accuracy_tools/msprobe/pytorch/monitor/module_spec_verifier.py deleted file mode 100644 index 72c35c90bf9540a31cfa1176274a3d2c66bc8946..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/module_spec_verifier.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -import abc -import torch - -from msprobe.pytorch.common.log import logger - -# 用于存储所有validator实现类的注册表 -config_validator_registry = {} - - -def register_config_validator(cls): - """装饰器 用于注册ConfigValidator的实现类""" - config_validator_registry[cls.__name__] = cls - return cls - - -class ConfigValidator(metaclass=abc.ABCMeta): - @abc.abstractmethod - def check_pattern_match(self, config_spec: str): - pass - - @abc.abstractmethod - def validate(self, actual_data, module_name: str, data_type: str, pattern_match): - pass - - -@register_config_validator -class TensorValidator(ConfigValidator): - def check_pattern_match(self, config_spec: str): - pattern = re.compile(r"tensor") - return pattern.match(config_spec) - - def validate(self, actual_data, module_name: str, data_type: str, pattern_match): - if not torch.is_tensor(actual_data): - raise ValueError( - f"Format of {module_name} {data_type} does not match the required format 'tensor' in config.") - - -@register_config_validator -class TupleValidator(ConfigValidator): - def check_pattern_match(self, config_spec: str): - pattern = re.compile(r"tuple\[(\d+)\]:?(\d+)?") - return pattern.match(config_spec) - - def validate(self, actual_data, module_name: str, data_type: str, pattern_match): - length, index = pattern_match.groups() - if index is None: - index = 0 - length, index = int(length), int(index) - - if not (0 <= index < length): - raise ValueError( - f"Format of {module_name} {data_type} in config.json does not match the required format 'tuple[x]:y'." - f"y must be greater than or equal to 0 and less than x.") - if not isinstance(actual_data, tuple): - raise ValueError( - f"Type of {module_name} {data_type} does not match spec of config.json, should be tuple, please check.") - if len(actual_data) != length: - raise ValueError( - f"Length of {module_name} {data_type} does not match spec of config.json, should be {length}, " - f"actual is {len(actual_data)} please check.") - return index - - -def validate_config_spec(config_spec: str, actual_data, module_name: str, data_type: str): - focused_col = None - if not config_spec or not isinstance(config_spec, str): - return focused_col - for _, validator_cls in config_validator_registry.items(): - config_validator = validator_cls() - pattern_match = config_validator.check_pattern_match(config_spec) - if pattern_match: - try: - focused_col = config_validator.validate(actual_data, module_name, data_type, pattern_match) - except ValueError as e: - logger.warning(f"config spec validate failed: {str(e)}") - return focused_col - logger.warning(f"config spec in {module_name} {data_type} not supported, " - f"expected spec:'tuple\[(\d+)\]:(\d+)' or 'tensor', actual spec: {config_spec}.") - return focused_col diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/optimizer_collect.py b/debug/accuracy_tools/msprobe/pytorch/monitor/optimizer_collect.py index 131f3ecab47e3935dfd8512b1b3e8ad479b3ba50..e074c78b8f8459fad00c2762fa87f8c093c182fe 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/optimizer_collect.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/optimizer_collect.py @@ -12,151 +12,120 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from collections import defaultdict +from abc import abstractmethod import torch import torch.distributed as dist from msprobe.pytorch.common.log import logger -from msprobe.pytorch.monitor.utils import MVResult, MVGradResult +from msprobe.pytorch.monitor.utils import MVResult +from msprobe.core.common.const import MonitorConst class OptimizerMon(object): - def __init__(self) -> None: + def __init__(self, torch_opt) -> None: self.fp16_to_fp32_param = {} - self.is_stage3 = False + self.torch_opt = torch_opt - def fetch_mv(self, monitor, torch_opt, params2name): - pass + def narrow_from_flatten(self, param, flatten_state): + return flatten_state + + def fetch_grad(self, monitor, params2name): + if not self.fp16_to_fp32_param: + self.map_fp16_to_fp32_param(self.torch_opt) - def _fetch_mv_in_adam(self, monitor, torch_opt, params2name): - exp_avg_dict = defaultdict(float) - exp_avg_sq_dict = defaultdict(float) - update_dict = defaultdict() - ratio_dict = defaultdict() + grad_dict = {} + first_param = True for param, name in params2name.items(): - if param in self.fp16_to_fp32_param: - param = self.fp16_to_fp32_param[param] - - if param in torch_opt.state: - state_param = torch_opt.state.get(param, None) - exp_avg = state_param.get("exp_avg", None) - exp_avg_sq = state_param.get("exp_avg_sq", None) - if exp_avg is None or exp_avg_sq is None: - logger.warning(f"exp_avg or exp_avg_sq of {name} is None, maybe something wrong happened.") - continue + if monitor.duplicate_param.get(name, False): + continue + if self.fp16_to_fp32_param and param not in self.fp16_to_fp32_param: + continue + grad = param.main_grad if monitor.params_have_main_grad else param.grad + element_in_cur_partition = self.fp16_to_fp32_param.get(param, param).numel() + if param.numel() != element_in_cur_partition: + if first_param: + grad = grad.flatten()[-element_in_cur_partition:] + else: # supposed to be the last one + grad = grad.flatten()[:element_in_cur_partition] + first_param = False + + if grad is None: + if not monitor.fsdp_wrapped_module: + logger.warning(f"grad is None: {name}, maybe something wrong happened.") + continue + tag = monitor.name2tag.get(name, {}).get(MonitorConst.POST_GRAD) + monitor.register_param_call_id("hook_optimizer", tag) + grad_dict[tag] = grad + return grad_dict + + def map_fp16_to_fp32_param(self, torch_opt): + pass + + def fetch_mv(self, monitor, params2name): + if not self.fp16_to_fp32_param: + self.map_fp16_to_fp32_param(self.torch_opt) + + exp_avg_dict = {} + exp_avg_sq_dict = {} + update_dict = {} + ratio_dict = {} + + if hasattr(self.torch_opt, 'state'): + state = self.torch_opt.state + elif hasattr(self.torch_opt, 'optimizer') and hasattr(self.torch_opt.optimizer, 'state'): + state = self.torch_opt.optimizer.state + else: + logger.warning('optimizer state can not accessed') + return MVResult(exp_avg=exp_avg_dict, exp_avg_sq=exp_avg_sq_dict, update=update_dict, ratio=ratio_dict) + + for lp_param, name in params2name.items(): + if lp_param in self.fp16_to_fp32_param: + hp_param = self.fp16_to_fp32_param[lp_param] + else: + hp_param = lp_param + + if hp_param in state: + state_param = state.get(hp_param, None) + exp_avg = self.narrow_from_flatten(lp_param, state_param.get("exp_avg", None)) + exp_avg_sq = self.narrow_from_flatten(lp_param, state_param.get("exp_avg_sq", None)) if monitor.mv_distribution: exp_avg_dict[name] = exp_avg exp_avg_sq_dict[name] = exp_avg_sq if monitor.mg_direction: exp_avg_dict[name] = exp_avg if monitor.ur_distribution: - if len(torch_opt.param_groups) > 1: - logger.info(f"the length of torch_opt.param_groups is {len(torch_opt.param_groups)}.") + if len(self.torch_opt.param_groups) > 1: + logger.info(f"the length of torch_opt.param_groups is {len(self.torch_opt.param_groups)}.") if 'step' in state_param: step = state_param['step'] # Optimizer from pytorch or FusedAdam from apex(used by megatron) - elif 'step' in torch_opt.param_groups[0]: - step = torch_opt.param_groups[0]['step'] # AdamW from mindspeed + elif 'step' in self.torch_opt.param_groups[0]: + step = self.torch_opt.param_groups[0]['step'] # AdamW from mindspeed else: logger.warning(f"step of {name} is None, maybe something wrong happened.") continue - exp_avg_hat = exp_avg / (1 - torch_opt.defaults['betas'][0] ** step) - exp_avg_sq_hat = exp_avg_sq / (1 - torch_opt.defaults['betas'][1] ** step) - update_dict[name] = exp_avg_hat / (torch.sqrt(exp_avg_sq_hat) + torch_opt.defaults['eps']) + exp_avg_hat = exp_avg / (1 - self.torch_opt.defaults['betas'][0] ** step) + exp_avg_sq_hat = exp_avg_sq / (1 - self.torch_opt.defaults['betas'][1] ** step) + update_dict[name] = exp_avg_hat / (torch.sqrt(exp_avg_sq_hat) + self.torch_opt.defaults['eps']) ratio_dict[name] = exp_avg_hat / torch.sqrt(exp_avg_sq_hat) monitor.update_heatmap_visualizer[name].pre_cal(update_dict[name]) monitor.ratio_heatmap_visualizer[name].pre_cal(ratio_dict[name]) return MVResult(exp_avg=exp_avg_dict, exp_avg_sq=exp_avg_sq_dict, update=update_dict, ratio=ratio_dict) - - def _fetch_mv_grad_in_adam(self, monitor, torch_opt, params2name, name2indices, fp32_partitioned_groups_flat): - exp_avg_dict = defaultdict(float) - exp_avg_sq_dict = defaultdict(float) - update_dict = defaultdict() - ratio_dict = defaultdict() - param2name = defaultdict() - fp32_partitioned_groups_flat_grad = defaultdict() - partition_id = dist.get_rank() - - def get_flatten_grad(self, optimizer, group_idx): - if fp32_partitioned_groups_flat[group_idx].grad is None: - if partition_id == dist.get_world_size() - 1 and not self.is_stage3: - fp32_partitioned_groups_flat_grad = optimizer.flatten_dense_tensors_aligned( - optimizer.averaged_gradients[group_idx], - int(optimizer.partition_size[group_idx]) - ).to(fp32_partitioned_groups_flat[group_idx].dtype) - else: - fp32_partitioned_groups_flat_grad = optimizer.flatten( - optimizer.averaged_gradients[group_idx] - ).to(fp32_partitioned_groups_flat[group_idx].dtype) - return fp32_partitioned_groups_flat_grad - else: - return fp32_partitioned_groups_flat[group_idx].grad - - for group_idx in range(len(fp32_partitioned_groups_flat)): - fp32_partitioned_groups_flat_grad[group_idx] = get_flatten_grad(self, torch_opt, group_idx) - - for name in params2name.values(): - start_idx, end_idx, group_idx, group_with_rank = name2indices[name] - if group_with_rank != partition_id and isinstance(group_with_rank, int): - continue - fp32_param = fp32_partitioned_groups_flat[group_idx][start_idx: end_idx] - fp32_param.grad = fp32_partitioned_groups_flat_grad[group_idx][start_idx: end_idx] - param2name[fp32_param] = name - if not torch_opt.state: - continue - state_param = list(torch_opt.state.values())[group_idx] - exp_avg = state_param.get("exp_avg", None) - exp_avg_sq = state_param.get("exp_avg_sq", None) - if exp_avg is None or exp_avg_sq is None: - logger.warning(f"exp_avg or exp_avg_sq of {name} is None, maybe something wrong happened.") - continue - exp_avg = exp_avg[start_idx: end_idx] - exp_avg_sq = exp_avg_sq[start_idx: end_idx] - if monitor.mv_distribution: - exp_avg_dict[name] = exp_avg - exp_avg_sq_dict[name] = exp_avg_sq - if monitor.mg_direction: - exp_avg_dict[name] = exp_avg - if monitor.ur_distribution: - if 'step' in state_param: - step = state_param['step'] # Optimizer from pytorch or FusedAdam from apex(used by megatron) - elif 'step' in torch_opt.param_groups[group_idx]: - step = torch_opt.param_groups[group_idx]['step'] # AdamW from mindspeed - else: - logger.warning(f"step of {name} is None, maybe something wrong happened.") - continue - exp_avg_hat = exp_avg / (1 - torch_opt.defaults['betas'][0] ** step) - exp_avg_sq_hat = exp_avg_sq / (1 - torch_opt.defaults['betas'][1] ** step) - update_dict[name] = exp_avg_hat / (torch.sqrt(exp_avg_sq_hat) + torch_opt.defaults['eps']) - ratio_dict[name] = exp_avg_hat / torch.sqrt(exp_avg_sq_hat) - monitor.update_heatmap_visualizer[name].pre_cal(update_dict[name]) - monitor.ratio_heatmap_visualizer[name].pre_cal(ratio_dict[name]) - del fp32_partitioned_groups_flat_grad - return MVGradResult(exp_avg=exp_avg_dict, exp_avg_sq=exp_avg_sq_dict, update=update_dict, ratio=ratio_dict, - grad=param2name) - + class MixPrecisionOptimizerMon(OptimizerMon): """ 混合精度优化器监控类。在混合精度训练中监控和管理优化器。 混合精度训练通过适当降低某些计算的精度来加速训练过程并减少内存消耗。 """ - - def map_fp16_tp_fp32_param(self, torch_opt): + def map_fp16_to_fp32_param(self, torch_opt): for fp16_group, fp32_group in zip(torch_opt.float16_groups, torch_opt.fp32_from_float16_groups): for fp16_param, fp32_param in zip(fp16_group, fp32_group): self.fp16_to_fp32_param[fp16_param] = fp32_param - def fetch_mv(self, monitor, torch_opt, params2name): - if not self.fp16_to_fp32_param and torch_opt is not None: - self.map_fp16_tp_fp32_param(torch_opt) - - return self._fetch_mv_in_adam(monitor, torch_opt, params2name) - class MegatronDistributedOptimizerMon(OptimizerMon): - def map_fp16_tp_fp32_param(self, torch_opt): + def map_fp16_to_fp32_param(self, torch_opt): if not (hasattr(torch_opt, "model_float16_groups") and hasattr(torch_opt, "shard_fp32_from_float16_groups")): raise Exception( @@ -167,184 +136,176 @@ class MegatronDistributedOptimizerMon(OptimizerMon): for fp16_param, shard_fp32_param in zip(fp16_group, shard_fp32_group): self.fp16_to_fp32_param[fp16_param] = shard_fp32_param - def fetch_mv(self, monitor, torch_opt, params2name): - if not self.fp16_to_fp32_param and torch_opt is not None: - self.map_fp16_tp_fp32_param(torch_opt) - - return self._fetch_mv_in_adam(monitor, torch_opt, params2name) - - -class MegatronFP32OptimizerMon(OptimizerMon): - def fetch_mv(self, monitor, torch_opt, params2name): - return self._fetch_mv_in_adam(monitor, torch_opt, params2name) - class MegatronChainedDistributedOptimizerMon(MegatronDistributedOptimizerMon): - def fetch_mv(self, monitor, torch_opt, params2name): - if not self.fp16_to_fp32_param and torch_opt is not None: - for opt in torch_opt.chained_optimizers: - self.map_fp16_tp_fp32_param(opt) + def map_fp16_to_fp32_param(self, torch_opt): + for opt in torch_opt.chained_optimizers: + super().map_fp16_to_fp32_param(opt) - if not isinstance(torch_opt, torch.optim.Optimizer) and not hasattr(torch_opt, 'state'): + if not hasattr(self.torch_opt, 'state'): torch_opt.state = {} - for opt in torch_opt.chained_optimizers: - torch_opt.state.update(opt.optimizer.state) - return self._fetch_mv_in_adam(monitor, torch_opt, params2name) + for opt in self.torch_opt.chained_optimizers: + self.torch_opt.state.update(opt.optimizer.state) class MegatronChainedMixPrecisionOptimizerMon(MixPrecisionOptimizerMon): - def fetch_mv(self, monitor, torch_opt, params2name): - if not self.fp16_to_fp32_param and torch_opt is not None: - for opt in torch_opt.chained_optimizers: - self.map_fp16_tp_fp32_param(opt) + def map_fp16_to_fp32_param(self, torch_opt): + for opt in torch_opt.chained_optimizers: + super().map_fp16_to_fp32_param(opt) - if not isinstance(torch_opt, torch.optim.Optimizer) and not hasattr(torch_opt, 'state'): + if not hasattr(self.torch_opt, 'state'): torch_opt.state = {} - for opt in torch_opt.chained_optimizers: - torch_opt.state.update(opt.optimizer.state) - return self._fetch_mv_in_adam(monitor, torch_opt, params2name) + for opt in self.torch_opt.chained_optimizers: + self.torch_opt.state.update(opt.optimizer.state) -class DeepSpeedZeroOptimizerStage0Mon(OptimizerMon): - def get_group_index(self, torch_opt): - bit16_groups = torch_opt.bf16_groups - param2group = defaultdict() - for group_idx, bit16_group in enumerate(bit16_groups): +class DeepSpeedZeroOptimizerMon(OptimizerMon): + """ + Base monitor class for DeepSpeed ZeRO optimizer. + ZeRO stage 0 no partition + ZeRO stage 1 partitions optimizer states across data parallel processes. + ZeRO stage 2 additionally partitions gradients. + ZeRO stage 3 additionally partitions parameters. + + This class provides monitoring capabilities for ZeRO optimizers by: + - Handling gradient collection for different ZeRO stages + - Managing optimizer state access for monitoring + """ + def __init__(self, torch_opt): + super().__init__(torch_opt) + self.stage = '' + self.bit16_groups = [] + self.fp32_flat_groups = [] + self.param2group = () + self.param2index = [] + self.group_offset = {} + + @abstractmethod + def get_grad_for_param(self, lp_param, group_idx, param_id): + raise NotImplementedError + + def param_not_in_partition(self, lp_param, group_idx): + param_slice_mapping = self.torch_opt.state_dict()['param_slice_mappings'][group_idx] + hp_address = param_slice_mapping.get(self.torch_opt.param_names.get(lp_param)) + return hp_address is None + + def get_position(self, lp_param, group_idx): + param_slice_mapping = self.torch_opt.state_dict()['param_slice_mappings'][group_idx] + hp_address = param_slice_mapping.get(self.torch_opt.param_names.get(lp_param)) + return hp_address.start, hp_address.numel + + def get_group_index(self): + param2group = {} + for group_idx, bit16_group in enumerate(self.bit16_groups): for param in bit16_group: param2group[param] = group_idx return param2group - - def fetch_mv(self, monitor, torch_opt, params2name, name2indices=None): - param2group = self.get_group_index(torch_opt) - exp_avg_dict = defaultdict(float) - exp_avg_sq_dict = defaultdict(float) - update_dict = defaultdict() - ratio_dict = defaultdict() - - param_slice_mappings = torch_opt.state_dict()['param_slice_mappings'] - for param, name in params2name.items(): - group_idx = param2group[param] - state = torch_opt.state[torch_opt.fp32_groups_flat_partition[group_idx]] - if state.get('exp_avg', None) is None: - logger.warning(f"optimizer state is None. Something is wrong if this is not the first step") - break - param_slice_mapping = param_slice_mappings[group_idx] - hp_address = param_slice_mapping.get(torch_opt.param_names[param]) - if hp_address is None: + + def get_param_index(self, lp_param, group_idx): + if not self.param2index: + for group in self.bit16_groups: + param2index = {} + for index, param in enumerate(group): + param2index[param] = index + self.param2index.append(param2index) + + return self.param2index[group_idx][lp_param] + + def narrow_from_flatten(self, param, flatten_state): + if flatten_state is None: + return flatten_state + group_idx = self.param2group[param] + if self.param_not_in_partition(param, group_idx): + return None + start, numel = self.get_position(param, group_idx) + return flatten_state.narrow(0, start, numel) + + def map_fp16_to_fp32_param(self, torch_opt): + for group_idx, group in enumerate(self.bit16_groups): + for param in group: + self.fp16_to_fp32_param[param] = self.fp32_flat_groups[group_idx] + + def fetch_grad(self, monitor, params2name): + grad_dict = {} + for lp_param, name in params2name.items(): + group_idx = self.param2group[lp_param] + param_id = self.get_param_index(lp_param, group_idx) + if self.param_not_in_partition(lp_param, group_idx): continue - start = hp_address.start - numel = hp_address.numel - - if monitor.mv_distribution: - exp_avg_dict[name] = state['exp_avg'].narrow(0, start, numel) - exp_avg_sq_dict[name] = state['exp_avg_sq'].narrow(0, start, numel) - if monitor.mg_direction: - exp_avg_dict[name] = state['exp'].narrow(0, start, numel) - if monitor.ur_distribution: - if len(torch_opt.param_groups) > 1: - logger.info(f"the length of torch_opt.param_groups is {len(torch_opt.param_groups)}.") - if 'step' in state: - step = state['step'] # Optimizer from pytorch or FusedAdam from apex(used by megatron) - elif 'step' in torch_opt.param_groups[0]: - step = torch_opt.param_groups[0]['step'] # AdamW from mindspeed + if self.stage == '1or2': + param_id = param_id - self.group_offset[group_idx] - 1 + grad = self.get_grad_for_param(lp_param, group_idx, param_id) + tag = monitor.name2tag.get(name, {}).get(MonitorConst.POST_GRAD) + monitor.register_param_call_id("hook_optimizer", tag) + grad_dict[tag] = grad + + return grad_dict + + +class DeepSpeedZeroOptimizerStage0Mon(DeepSpeedZeroOptimizerMon): + def __init__(self, torch_opt): + super().__init__(torch_opt) + self.stage = '0' + self.bit16_groups = torch_opt.bf16_groups + self.fp32_flat_groups = torch_opt.fp32_groups_flat_partition + self.param2group = self.get_group_index() + + def get_grad_for_param(self, lp_param, group_idx, param_id): + return self.torch_opt.fp32_groups_gradient_dict[group_idx][param_id] + + +class DeepSpeedZeroOptimizerStage1or2Mon(DeepSpeedZeroOptimizerMon): + def __init__(self, torch_opt): + super().__init__(torch_opt) + self.stage = '1or2' + self.bit16_groups = torch_opt.bit16_groups + self.fp32_flat_groups = torch_opt.single_partition_of_fp32_groups + self.param2group = self.get_group_index() + self.group_offset = {} + self.get_group_offset() + + def get_grad_for_param(self, lp_param, group_idx, param_id): + if getattr(self.torch_opt, "cpu_offload", False): + grads = self.torch_opt.single_partition_of_fp32_groups[group_idx].grad + start, numel = self.get_position(lp_param, group_idx) + grad = grads.narrow(0, start, numel) + else: + grad = self.torch_opt.averaged_gradients[group_idx][param_id] + return grad + + def get_group_offset(self): + for group_idx, group in enumerate(self.bit16_groups): + self.group_offset[group_idx] = -1 + for lp_param in group: + if self.param_not_in_partition(lp_param, group_idx): + self.group_offset[group_idx] = self.get_param_index(lp_param, group_idx) else: - logger.warning(f"step of {name} is None, maybe something wrong happened.") - continue - exp_avg = state['exp_avg'].narrow(0, start, numel) - exp_avg_sq = state['exp_avg_sq'].narrow(0, start, numel) - exp_avg_hat = exp_avg / (1 - torch_opt.defaults['betas'][0] ** step) - exp_avg_sq_hat = exp_avg_sq / (1 - torch_opt.defaults['betas'][1] ** step) - update_dict[name] = exp_avg_hat / (torch.sqrt(exp_avg_sq_hat) + torch_opt.defaults['eps']) - ratio_dict[name] = exp_avg_hat / torch.sqrt(exp_avg_sq_hat) - monitor.update_heatmap_visualizer[name].pre_cal(update_dict[name]) - monitor.ratio_heatmap_visualizer[name].pre_cal(ratio_dict[name]) - return MVResult(exp_avg=exp_avg_dict, exp_avg_sq=exp_avg_sq_dict, update=update_dict, ratio=ratio_dict) - + break -class DeepSpeedZeroOptimizerStage3Mon(OptimizerMon): - def get_param_index(self, params2name, name2index, torch_opt): - fp16_groups = torch_opt.fp16_partitioned_groups - name2indices = defaultdict() - index_length = defaultdict() - index = 0 - idx = 0 - for group_idx, fp16_group in enumerate(fp16_groups): - for param in fp16_group: - param_length = len(param.flatten()) - index_length[idx] = (index, index + param_length, group_idx) - index += param_length - idx += 1 - for _, name in params2name.items(): - idx = name2index[name] - start_idx, end_idx, group_idx = index_length[idx] - name2indices[name] = (start_idx, end_idx, group_idx, None) - return name2indices - - def fetch_mv(self, monitor, torch_opt, params2name, name2indices=None): - self.is_stage3 = True - fp32_partitioned_groups_flat = torch_opt.fp32_partitioned_groups_flat - return self._fetch_mv_grad_in_adam(monitor, torch_opt, params2name, name2indices, fp32_partitioned_groups_flat) - - -class DeepSpeedZeroOptimizerStage1or2Mon(OptimizerMon): - @staticmethod - def get_group_index(fp32_length, world_size, index): - for i in range(len(fp32_length) - 1): - if fp32_length[i] <= index < fp32_length[i + 1]: - interval_start = fp32_length[i] - interval_length = fp32_length[i + 1] - fp32_length[i] - sub_interval_length = interval_length // world_size - sub_index = (index - interval_start) // sub_interval_length - sub_interval_start = interval_start + sub_index * sub_interval_length - return sub_interval_start, min(sub_index, world_size - 1) - return fp32_length[-1], 0 - - def get_param_index(self, params2name, name2index, torch_opt): - padding = torch_opt.groups_padding - world_size = dist.get_world_size() - fp32_length = [0] - for fp32_group_index, single_partition_of_fp32_group in enumerate(torch_opt.single_partition_of_fp32_groups): - fp32_length.append(len(single_partition_of_fp32_group) * world_size + fp32_length[fp32_group_index]) - - bf16_groups = [] - name2indices = defaultdict() - index_length = defaultdict() - index = 0 - idx = 0 - for group_idx, bf16_group in enumerate(torch_opt.bit16_groups): - bf16_groups.extend(bf16_group) - for param in bf16_group: - param_length = len(param.flatten()) - group_index, group_with_rank = self.get_group_index(fp32_length, world_size, index) - index_length[idx] = (index, index + param_length, group_idx, group_index, group_with_rank) - index += param_length - idx += 1 - group_length = len(bf16_groups) / len(torch_opt.bit16_groups) - for _, name in params2name.items(): - name_index = name2index[name] - start_idx, end_idx, group_idx, group_index, group_with_rank = index_length[name_index] - need_padding = True if group_with_rank == world_size - 1 else False - new_start_idx = start_idx - group_index - new_end_idx = end_idx - group_index - if need_padding and group_length - 1 <= name_index <= len(bf16_groups) - 1 and name_index % ( - group_length - 1) == 0: - new_end_idx -= padding[int(name_index // (group_length - 1) - 1)] - name2indices[name] = (new_start_idx, new_end_idx, group_idx, group_with_rank) - return name2indices - - def fetch_mv(self, monitor, torch_opt, params2name, name2indices=None): - fp32_partitioned_groups_flat = torch_opt.single_partition_of_fp32_groups - return self._fetch_mv_grad_in_adam(monitor, torch_opt, params2name, name2indices, fp32_partitioned_groups_flat) - - -class DummyOptimizerMon(OptimizerMon): - def fetch_mv(self, monitor, torch_opt, params2name): - return self._fetch_mv_in_adam(monitor, torch_opt, params2name) + +class DeepSpeedZeroOptimizerStage3Mon(DeepSpeedZeroOptimizerMon): + def __init__(self, torch_opt): + super().__init__(torch_opt) + self.stage = '3' + self.bit16_groups = torch_opt.fp16_groups + self.fp32_flat_groups = torch_opt.fp32_partitioned_groups_flat + self.param2group = self.get_group_index() + + def param_not_in_partition(self, param, group_index): + """Each param partioned across all zero ranks""" + return False + + def get_position(self, lp_param, group_idx): + param_id = self.torch_opt.get_param_id(lp_param) + return self.torch_opt.grad_position[param_id][1:] + + def get_grad_for_param(self, lp_param, group_idx, param_id): + return self.torch_opt.averaged_gradients[group_idx][param_id] class OptimizerMonFactory: _optimizer_mon_map = { - "FP32Optimizer": MegatronFP32OptimizerMon, + "FP32Optimizer": OptimizerMon, "Float16OptimizerWithFloat16Params": MixPrecisionOptimizerMon, "DistributedOptimizer": MegatronDistributedOptimizerMon, "ChainedDistributedOptimizer": MegatronChainedDistributedOptimizerMon, @@ -352,7 +313,7 @@ class OptimizerMonFactory: "BF16_Optimizer": DeepSpeedZeroOptimizerStage0Mon, "DeepSpeedZeroOptimizer": DeepSpeedZeroOptimizerStage1or2Mon, "DeepSpeedZeroOptimizer_Stage3": DeepSpeedZeroOptimizerStage3Mon, - "Adam": DummyOptimizerMon + "Adam": OptimizerMon } @staticmethod @@ -361,6 +322,7 @@ class OptimizerMonFactory: optimizer_class = optimizer.__class__.__name__ if optimizer_class == "ChainedOptimizer": optimizer_class = "Chained" + optimizer.chained_optimizers[0].__class__.__name__ + logger.info(f'The optimizer type is {optimizer_class}') - optimizer_mon_class = OptimizerMonFactory._optimizer_mon_map.get(optimizer_class, DummyOptimizerMon) - return optimizer_mon_class(), optimizer_class + optimizer_mon_class = OptimizerMonFactory._optimizer_mon_map.get(optimizer_class, OptimizerMon) + return optimizer_mon_class(optimizer) diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/unittest/test_monitor.py b/debug/accuracy_tools/msprobe/pytorch/monitor/unittest/test_monitor.py deleted file mode 100644 index 4d5c1a717d80ee30414f25b44a93ddc7257ef2c7..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/unittest/test_monitor.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -import os -import re -from glob import glob - -import pandas as pd - -from msprobe.pytorch.common.log import logger - - -def parse_logfile(logfile): - grad_norm = [] - step = [] - with open(logfile) as f: - for line in f.readlines(): - if 'consumed samples' in line: - grad_norm.append(float(re.findall('(?<=grad norm\: )[\d\.]*', line)[0])) - return grad_norm - - -def parse_monitor_output(output_dir): - reduced = {} - unreduced = {} - for directory in glob(output_dir + '*'): - rank = int(re.findall('(?<=rank)[\d]*', directory)[0]) - unreduced[rank] = [] - reduced[rank] = [] - for file in os.listdir(directory): - df = pd.read_csv(os.path.join(directory, file)) - if '_unreduced_' in file: - unreduced[rank].append(df) - pass - elif '_reduced_' in file: - reduced[rank].append(df) - else: - logger.info(f'unexpected file {file} in {directory}') - return reduced, unreduced - - -def valid_reduce(reduced, unreduced, tp_size, dp_size, sequence_parallel): - steps = len(reduced[0]) - world_size = len(reduced) - errors = [] - for _, row in unreduced[0][0].iterrows(): - param = row['param_name'] - is_tp_duplicate = False - for step in range(2): - # sum reduced - reduced_mean = 0. - for rank in range(world_size): - if len(reduced[rank]) == 0: - continue - df = reduced[rank][step] - value = list(df[df['param_name'] == param]['mean']) - if not value: - if step == 0: - is_tp_duplicate = True - continue - reduced_mean += value[0] - - # sum unreduced - unreduced_mean = 0. - for rank in range(world_size): - df = unreduced[rank][step] - value = list(df[df['param_name'] == param]['mean']) - if not value: - continue - unreduced_mean += list(df[df['param_name'] == param]['mean'])[0] - - unreduced_mean /= dp_size - if is_tp_duplicate and (not sequence_parallel or 'embedding' in param): - unreduced_mean /= tp_size - try: - assert_equal(unreduced_mean, reduced_mean) - except AssertionError as e: - errors.append([param, step, e, is_tp_duplicate]) - if errors: - logger.info(errors) - else: - logger.info(f'grad mean is in consist between unreduced grad and reduced grad monitord.') - - -def assert_equal(a, b): - if b == 0 or a == 0: - return - if b == 0: - rel_diff = a - elif a == 0: - rel_diff = b - else: - rel_diff = abs(a / b - 1) - assert rel_diff < 0.01, f'{a}, {b}, {rel_diff}' - - -def valid_total_norm(total_norm, reduced, duplicate_embedding): - steps = len(total_norm) - world_size = len(reduced) - errors = [] - for step in range(steps): - calculated_norm = 0. - for rank in range(world_size): - if len(reduced[rank]) == 0: - if step == 0: - logger.info(f'rank {rank} is duplicated in dp group') - continue - for _, row in reduced[rank][step].iterrows(): - if duplicate_embedding and 'word_embedding' in row['param_name']: - continue - calculated_norm += row['norm'] ** 2 - try: - assert_equal(calculated_norm ** 0.5, total_norm[step]) - except AssertionError as e: - errors.append([step, e]) - if errors: - logger.info('total norm errors: ', errors) - else: - logger.info('grad norm in consist between training log and reduced gradients monitored') - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--monitor_output', '-m', type=str, required=True, - help='path prefix to the output of monitor e.g. monitor_output/Aug12_07-16') - parser.add_argument('--logfile', '-l', type=str, required=True, help='path to the training log file') - parser.add_argument('--tp_size', '-t', type=int, required=True, help='tp parallel size') - parser.add_argument('--dp_size', '-d', type=int, required=True, help='dp parallel size') - parser.add_argument('--pp_size', '-p', type=int, required=True, help='pp parallel size') - parser.add_argument('--untie_embeddings_and_output_weights', '-u', action="store_true", default=False, - help='whether untie_embeddings_and_output_weights in pp parallel') - parser.add_argument('--sequence_parallel', '-s', action="store_true", default=False, - help='whether sequence parallel is enabled. Add -s to store true') - - args = parser.parse_args() - - assert args.tp_size > 0, 'if tp not enabled, set tp_size = 1' - assert args.dp_size > 0, 'if tp not enabled, set dp_size = 1' - assert args.pp_size > 0, 'if tp not enabled, set pp_size = 1' - - total_norm = parse_logfile(args.logfile) - reduced, unreduced = parse_monitor_output(args.monitor_output) - - duplicate_embedding = not args.untie_embeddings_and_output_weights and args.pp_size > 1 - - valid_total_norm(total_norm, reduced, duplicate_embedding) - valid_reduce(reduced, unreduced, args.tp_size, args.dp_size, args.sequence_parallel) diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/utils.py b/debug/accuracy_tools/msprobe/pytorch/monitor/utils.py index 94afe56ffcfe7571a189c5f6959b2eb9a2779d81..e06f08417bdb54675b16d908de3136a0f378dc64 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/utils.py @@ -22,10 +22,10 @@ import re import torch -from msprobe.core.common.const import MonitorConst, Const +from msprobe.core.common.const import MonitorConst from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import is_int -from msprobe.core.common.file_utils import check_file_or_directory_path +from msprobe.core.common.file_utils import check_file_or_directory_path, recursive_chmod device = "cpu" @@ -43,7 +43,6 @@ DIRECTORY_MAX_LENGTH = 4096 beijing_tz = timezone(timedelta(hours=8)) MVResult = namedtuple('MVResult', ("exp_avg", "exp_avg_sq", "update", "ratio")) -MVGradResult = namedtuple('MVGradResult', ("exp_avg", "exp_avg_sq", "update", "ratio", "grad")) class MsgConst: @@ -102,9 +101,21 @@ def validate_ops(ops): default_op = MonitorConst.OP_LIST[0] valid_ops.append(default_op) logger.info_on_rank_0(f"There is no valid ops, default op {default_op} is used") + # 增加默认shape和dtype参数 + if "shape" not in valid_ops and "dtype" not in valid_ops: + valid_ops.extend(["shape", "dtype"]) return valid_ops +def validate_ndigits(ndigits): + if not ndigits: + return + if not is_int(ndigits) or ndigits <= 0: + raise ValueError(f"ndigits({ndigits}) is not a positive integer, current is: {ndigits}.") + if ndigits > MonitorConst.MAX_NDIGITS: + raise ValueError(f"The maximum supported ndigits is {MonitorConst.MAX_NDIGITS}, current value: {ndigits}.") + + def validate_ranks(ranks): if not isinstance(ranks, list): raise TypeError("module_ranks should be a list") @@ -190,7 +201,7 @@ def validate_alert(alert): args = rule.get("args") if args and isinstance(args, dict): threshold = args.get("threshold") - if not isinstance(threshold, float) or threshold < 0: + if not isinstance(threshold, (float, int)) or threshold < 0: raise TypeError('threshold must be float and not less than 0') dump = alert.get('dump') if dump and not isinstance(dump, bool): @@ -206,9 +217,17 @@ def validate_step_count_per_record(step_count_per_record): raise ValueError("step_count_per_record must smaller than 1e6") +def validate_dynamic_on(dynamic_on): + if not isinstance(dynamic_on, bool): + raise TypeError('dynamic_on should be a bool') + + def validate_config(config): config['ops'] = validate_ops(config.get('ops', [])) + ndigits = config.get('ndigits') + validate_ndigits(ndigits) + eps = config.get('eps', 1e-8) if not isinstance(eps, float): raise TypeError("eps should be a float") @@ -246,9 +265,20 @@ def validate_config(config): step_count_per_record = config.get('step_count_per_record', 1) validate_step_count_per_record(step_count_per_record) + config["start_step"] = validate_int_arg(config.get("start_step"), "start_step", + MonitorConst.DEFAULT_START_STEP, MonitorConst.DEFAULT_START_STEP) + config["collect_times"] = validate_int_arg(config.get("collect_times"), "collect_times", + MonitorConst.DEFAULT_MIN_COLLECT_TIMES, + MonitorConst.DEFAULT_MAX_COLLECT_TIMES) + config["step_interval"] = validate_int_arg(config.get("step_interval"), "step_interval", + MonitorConst.DEFAULT_STEP_INTERVAL, MonitorConst.DEFAULT_STEP_INTERVAL) + squash_name = config.get('squash_name', True) validate_squash_name(squash_name) + dynamic_on = config.get('dynamic_on', False) + validate_dynamic_on(dynamic_on) + if not targets: if xy_distribution: config["all_xy"] = True @@ -257,6 +287,8 @@ def validate_config(config): def time_str2time_digit(time_str): time_format = '%b%d_%H-%M-%S' + if not isinstance(time_str, str): + raise TypeError(f"time_str:{time_str} should be a str") try: time_digit = datetime.strptime(time_str, time_format) except Exception as e: @@ -284,3 +316,40 @@ def get_target_output_dir(monitor_path, time_start, time_end): if start_ok and end_ok: result[rank] = os.path.join(monitor_path, dirname) return result + + +def chmod_tensorboard_dir(path): + """ + format配置为tensorboard时，需要补充文件权限设置 + """ + try: + recursive_chmod(path) + except Exception as e: + logger.warning(f"chmod tensorboard dir wrong because {e}, not updated, please check!!!") + + +def validate_set_monitor(grad_acc_steps, start_iteration): + """ + validate parameters of set_monitor. + """ + grad_acc_steps = validate_int_arg(grad_acc_steps, "grad_acc_steps", + MonitorConst.DEFAULT_GRAD_ACC_STEPS, MonitorConst.DEFAULT_GRAD_ACC_STEPS) + + start_iteration = validate_int_arg(start_iteration, "start_iteration", + MonitorConst.DEFAULT_START_ITERATION, MonitorConst.DEFAULT_START_ITERATION) + return grad_acc_steps, start_iteration + + +def validate_int_arg(value, name, minimum, default_value): + """Validate int args, if any exception occurs, use the default value.""" + if value is None: + return default_value + try: + if not is_int(value): + raise TypeError(f"{name} must be int") + if value < minimum: + raise ValueError(f"{name} must greater than {minimum}") + except Exception as e: + value = default_value + logger.warning(f"Validate {name} failed, {e}, replaced with default value {value}.") + return value diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py index 7a265e70fa4cbe95c897c35d68e4afa8ebd77249..18d8e0f1d0ab00fb723eafa9d0dc17d92bd164a6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py @@ -125,8 +125,6 @@ class Saver: def write_summary_csv(self, test_result): test_rows = [] - if self.stack_info: - test_rows[0].append(self.COLUMN_STACK_INFO) check_op_str_pattern_valid(test_result.api_name) df_row = [test_result.api_name, test_result.is_fwd_success, test_result.is_bwd_success] diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index b9201cfaac74e38bbbaee468b6c452895f8b38f9..916a68aece20ba620877004d25b15bbbcc01c41e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -16,6 +16,7 @@ import json import os import time +import multiprocessing from multiprocessing import Pool import torch @@ -52,6 +53,7 @@ class PtdbgDispatch(TorchDispatchMode): return if dump_path is None: logger.error("Please set dump_path when dump_mode is config!") + raise DispatchException("Please set dump_path when dump_mode is config!") check_file_or_directory_path(dump_path, True) self.device_id = torch_npu._C._npu_getDevice() @@ -85,6 +87,11 @@ class PtdbgDispatch(TorchDispatchMode): self.get_ops(yaml_path) self.lock = None + max_process_num = max(int((multiprocessing.cpu_count() + 1) // Const.CPU_QUARTER), 1) + if process_num > max_process_num: + logger.error(f"process_num should be less than or equal to {max_process_num}, but got {process_num}!") + raise DispatchException(f'process_num should be less than or equal to {max_process_num}, ' + f'but got {process_num}!') if process_num > 0: self.pool = Pool(process_num) if debug: @@ -115,6 +122,8 @@ class PtdbgDispatch(TorchDispatchMode): if len(json_line_data) == 0: break msg = json.loads(json_line_data) + if len(msg) < 2: + raise ValueError("JSON data does not contain enough elements. Expected at least 2 elements.") self.all_summary[msg[0]] = msg[1] fp_handle.close() @@ -199,8 +208,10 @@ class PtdbgDispatch(TorchDispatchMode): dispatch_workflow(run_param, data_info) else: self.lock.acquire() - self.all_summary.append([]) - self.lock.release() + try: + self.all_summary.append([]) + finally: + self.lock.release() run_param.process_flag = True if self.check_fun(func, run_param): data_info = DisPatchDataInfo(cpu_args, cpu_kwargs, self.all_summary, None, npu_out_cpu, cpu_out, diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py index a154064755ed116eeb2f2ea97b50160bf4b7beb9..dbf7626a2710a3f10ddc8d45795988b89081d0d5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py @@ -20,7 +20,7 @@ from datetime import datetime, timezone import torch from msprobe.core.common.const import Const -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.core.common.file_utils import FileOpen, save_npy, save_json from msprobe.pytorch.common.log import logger @@ -110,8 +110,11 @@ def dump_data(data, prefix, dump_path): def save_temp_summary(api_index, single_api_summary, path, lock): summary_path = os.path.join(path, f'summary.json') lock.acquire() - data = [api_index, single_api_summary] - save_json(summary_path, data, mode='a') + try: + data = [api_index, single_api_summary] + save_json(summary_path, data, mode='a') + finally: + lock.release() def dispatch_workflow(run_param: DispatchRunParam, data_info: DisPatchDataInfo): diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index 2116186cc046865388c40d33142384301f84acd2..37105551a3bccca548fe2b6594f4848324746b49 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -27,7 +27,7 @@ else: pta_cpu_device = torch.device("cpu") from msprobe.core.common.const import CompareConst -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator from msprobe.pytorch.common.log import logger diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/interactive_cli.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/interactive_cli.py index ac6f3d234e3a6681a580f16e56d94204223102f1..7f08b7929cd46961cb5850f16aa6ad7d7eace533 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/interactive_cli.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/interactive_cli.py @@ -45,12 +45,7 @@ class InteractiveCli(cmd.Cmd): @catch_exception def default(self, line=""): - self.util.execute_command(line) - return False - - @catch_exception - def do_run(self, line=""): - self.util.execute_command(line) + self.stdout.write("Command invalid, Only support command start with cad/vc/dc/pk/cn/pt\n") @catch_exception def do_vc(self, line=""): diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py index db731b338244ca78bfd460633a7442b1e2ef2d5d..144e886cab92bac6948a0c8efb0510ed30ea0700 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import hashlib import os import re import subprocess import sys import time +import zlib from collections import namedtuple import numpy as np @@ -114,11 +114,12 @@ class Util: @staticmethod def get_md5_for_numpy(obj): np_bytes = obj.tobytes() - md5_hash = hashlib.md5(np_bytes) - return md5_hash.hexdigest() + md5_crc = zlib.crc32(np_bytes) + return f"{md5_crc:08x}" @staticmethod def deal_with_dir_or_file_inconsistency(output_path): + logger.warning(f"Trying to delete {output_path}") remove_path(output_path) raise ParseException("Inconsistent directory structure or file.") diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index f63f467213a53f1317c1ca842b4801e1c12119b9..2ddfaf7b3012292da7231f6a0ebd42d6d6b7d92a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -43,6 +43,7 @@ class TensorConfig(BaseConfig): self.tls_path = json_config.get("tls_path", "./") self.online_run_ut_recompute = json_config.get("online_run_ut_recompute", False) self.check_config() + self._check_summary_mode() self._check_file_format() if self.online_run_ut: self._check_online_run_ut() @@ -67,6 +68,7 @@ class TensorConfig(BaseConfig): check_file_or_directory_path(os.path.join(self.tls_path, "client.key")) check_file_or_directory_path(os.path.join(self.tls_path, "client.crt")) check_crt_valid(os.path.join(self.tls_path, "client.crt")) + check_crt_valid(os.path.join(self.tls_path, "client.key"), True) if not isinstance(self.host, str) or not re.match(Const.ipv4_pattern, self.host): raise Exception(f"host: {self.host} is invalid.") @@ -81,9 +83,8 @@ class StatisticsConfig(BaseConfig): self.check_config() self._check_summary_mode() - def _check_summary_mode(self): - if self.summary_mode and self.summary_mode not in ["statistics", "md5"]: - raise Exception("summary_mode is invalid") + self.tensor_list = json_config.get("tensor_list", []) + self._check_str_list_config(self.tensor_list, "tensor_list") class OverflowCheckConfig(BaseConfig): @@ -96,6 +97,8 @@ class OverflowCheckConfig(BaseConfig): def check_overflow_config(self): if self.overflow_nums is not None and not is_int(self.overflow_nums): raise Exception("overflow_num is invalid") + if self.overflow_nums is not None and self.overflow_nums != -1 and self.overflow_nums <= 0: + raise Exception("overflow_nums should be -1 or positive integer") if self.check_mode is not None and self.check_mode not in ["all", "aicore", "atomic"]: raise Exception("check_mode is invalid") @@ -149,7 +152,7 @@ class FreeBenchmarkCheckConfig(BaseConfig): self.pert_mode in PytorchFreeBenchmarkConst.CPU_MODE_LIST ): msg = ( - f"You neet to and can only set fuzz_device as {DeviceType.CPU} " + f"You need to and can only set fuzz_device as {DeviceType.CPU} " f"when pert_mode in {PytorchFreeBenchmarkConst.CPU_MODE_LIST}" ) logger.error_log_with_exp( diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index b0b2780328d0261b41fab005d70e37b4168aceb7..7fa9e2d8a6ab71d69936e59cf9af6f88aa98b41f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -15,31 +15,32 @@ import functools import os -from collections import namedtuple, defaultdict +from collections import defaultdict import torch + from msprobe.core.common.const import Const from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.file_utils import create_directory -from msprobe.core.common.utils import print_tools_ends_info, DumpPathAggregation +from msprobe.core.common.utils import print_tools_ends_info, DumpPathAggregation, replace_last_occurrence from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs from msprobe.core.data_dump.scope import BaseScope +from msprobe.core.data_dump.api_registry import ApiRegistry from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.utils import get_rank_if_initialized, is_recomputation from msprobe.pytorch.dump.kernel_dump.kernel_config import create_kernel_config_json from msprobe.pytorch.dump.module_dump.module_processer import ModuleProcesser -from msprobe.pytorch.hook_module.api_register import get_api_register +from msprobe.pytorch.hook_module.api_register import get_api_register, ApiTemplate from msprobe.pytorch.hook_module.hook_module import HOOKModule +from msprobe.pytorch.hook_module.jit_script_wrapper import wrap_jit_script_func from msprobe.pytorch.hook_module.register_optimizer_hook import register_optimizer_hook torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' if torch_version_above_or_equal_2: from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.dump_dispatch import run_ut_dispatch -HookFn = namedtuple('hookFn', ['pre_hook', 'forward_hook', 'backward_hook', 'forward_hook_torch_version_below_2']) - class Service: def __init__(self, config): @@ -52,6 +53,7 @@ class Service: self.current_iter = 0 self.loop = 0 self.init_step = 0 + self.cur_token_id = 0 self.first_start = True self.current_rank = None self.dump_iter_dir = None @@ -62,25 +64,27 @@ class Service: # 提前注册，确保注册尽可能多的API hook self.api_register = get_api_register() self.register_api_hook() - self.init_for_debug_level() + self.currrent_step_first_debug_save = True + self.debug_variable_counter = None + self.ori_customer_func = {} def build_hook(self, module_type, name): - def pre_hook(api_or_module_name, module, args, kwargs): - if not self.should_execute_hook(module_type, module, True): - return args, kwargs + def pre_hook(api_or_module_name, module, args, kwargs=None): + kwargs = {} if kwargs is None else kwargs + + if module_type == BaseScope.Module_Type_Module or \ + not self.should_execute_hook(module_type, module, True): + return is_recompute = is_recomputation() self.inner_switch = True - if module_type == BaseScope.Module_Type_Module: - api_or_module_name = module.mindstudio_reserved_name[-1] - else: - module.forward_data_collected = True - HOOKModule.add_module_count(name) + module.forward_data_collected = True + HOOKModule.add_module_count(name) self.data_collector.update_api_or_module_name(api_or_module_name) if self.config.online_run_ut: self.inner_switch = False - return None, None + return if self.data_collector: module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=None) self.data_collector.forward_input_data_collect( @@ -92,7 +96,6 @@ class Service: ) self.inner_switch = False - return args, kwargs def grad_hook(module, ori_name, param_name): def hook_fn(grad): @@ -139,10 +142,12 @@ class Service: # 记录当前模块的参数梯度信息已占位 self.params_grad_info[grad_name] = True - def forward_hook(api_or_module_name, module, args, kwargs, output): + def forward_hook(api_or_module_name, module, args, kwargs_or_output, output_or_kwargs=None): if not self.should_execute_hook(module_type, module, True): return None is_recompute = is_recomputation() + kwargs = kwargs_or_output if torch_version_above_or_equal_2 else {} + output = output_or_kwargs if torch_version_above_or_equal_2 else kwargs_or_output self.inner_switch = True if self.config.online_run_ut: @@ -162,9 +167,8 @@ class Service: return None module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=output) + self.data_collector.update_api_or_module_name(api_or_module_name) if module_type == BaseScope.Module_Type_Module: - api_or_module_name = module.mindstudio_reserved_name[-1] - self.data_collector.update_api_or_module_name(api_or_module_name) params_dict = {} if self.config.task != Const.STRUCTURE: params_dict = { @@ -188,7 +192,6 @@ class Service: ) init_params_grad_info(module, params_dict) else: - self.data_collector.update_api_or_module_name(api_or_module_name) self.data_collector.forward_output_data_collect( api_or_module_name, module, @@ -204,17 +207,12 @@ class Service: self.inner_switch = False return output - def forward_hook_torch_version_below_2(api_or_module_name, module, args, output): - return forward_hook(api_or_module_name, module, args, {}, output) - def backward_hook(api_or_module_name, module, grad_input, grad_output): if not self.should_execute_hook(module_type, module, False): return is_recompute = is_recomputation() self.inner_switch = True - if module_type == BaseScope.Module_Type_Module: - api_or_module_name = module.mindstudio_reserved_name[-1] self.data_collector.update_api_or_module_name(api_or_module_name) if self.config.online_run_ut: @@ -234,21 +232,39 @@ class Service: self.inner_switch = False pid = os.getpid() - full_forward_name = None - full_backward_name = None + full_forward_name = name if module_type == BaseScope.Module_Type_API: full_forward_name = name + str(HOOKModule.get_module_count(name)) + Const.SEP + Const.FORWARD - full_backward_name = name + str(HOOKModule.get_module_count(name)) + Const.SEP + Const.BACKWARD + full_backward_name = replace_last_occurrence(full_forward_name, Const.FORWARD, Const.BACKWARD) pre_forward_hook_fn = functools.partial(pre_hook, full_forward_name) forward_hook_fn = functools.partial(forward_hook, full_forward_name) backward_hook_fn = functools.partial(backward_hook, full_backward_name) - forward_hook_torch_version_below_2_fn = functools.partial( - forward_hook_torch_version_below_2, - full_forward_name - ) - return HookFn(pre_forward_hook_fn, forward_hook_fn, backward_hook_fn, forward_hook_torch_version_below_2_fn) - def start(self, model): + return pre_forward_hook_fn, forward_hook_fn, backward_hook_fn + + def register_infer_count_hook(self, root_model, token_range): + """ + 通过root_model执行的轮次来判断当前在第几个token + param root_model: 需要采集的推理模型 + param token_range: [start, end], 采集infer的token循环范围，左右皆包含在内 + return: None + """ + def infer_hook(model, args): + if self.cur_token_id == token_range[0]: + self.switch = True + logger.info(f"Current token id: {self.cur_token_id}, start dump infer token.") + elif token_range[0] < self.cur_token_id <= token_range[1]: + logger.debug(f"Current token id: {self.cur_token_id}.") + elif self.cur_token_id == token_range[1] + 1: + self.switch = False + logger.info(f"Current token id: {self.cur_token_id}, exceed token_range, early stop dump infer token.") + self.cur_token_id += 1 + if isinstance(root_model, list): + root_model = root_model[0] + logger.warning("Infer model can only input one to support token_range, choose the first one.") + root_model.register_forward_pre_hook(infer_hook) + + def start(self, model, token_range=None): self.current_iter = self.loop + self.init_step self.data_collector.update_iter(self.current_iter) if self.config.level == Const.LEVEL_DEBUG: @@ -257,6 +273,7 @@ class Service: return self.model = model + self.cur_token_id = 0 if self.first_start: try: self.current_rank = get_rank_if_initialized() @@ -266,13 +283,19 @@ class Service: if self.config.rank and self.current_rank not in self.config.rank: return + self.register_module_hook() if self.config.level == Const.LEVEL_MIX: register_optimizer_hook(self.data_collector) self.first_start = False + + if token_range: + self.register_infer_count_hook(self.model, token_range) + if self.config.online_run_ut and torch_version_above_or_equal_2: run_ut_dispatch(self.attl, True, self.config.online_run_ut_recompute) - self.switch = True + if token_range is None: + self.switch = True logger.info_on_rank_0(f"Dump switch is turned on at step {self.current_iter}. ") if not self.config.online_run_ut: self.create_dirs() @@ -293,23 +316,18 @@ class Service: if self.config.online_run_ut and torch_version_above_or_equal_2: run_ut_dispatch(self.attl, False, self.config.online_run_ut_recompute) return - if self.config.async_dump: - self.data_collector.fill_stack_tensor_data() - if self.config.task == Const.TENSOR: - self.data_collector.data_processor.dump_async_data() + if self.config.async_dump and self.config.task in [Const.STATISTICS, Const.TENSOR]: + self.data_collector.data_processor.dump_async_data() self.data_collector.write_json() def step(self): - if self.config.level == Const.LEVEL_DEBUG: - return if self.should_stop_service: return - if self.config.async_dump: - self.data_collector.fill_stack_tensor_data() - if self.config.task == Const.TENSOR: - self.data_collector.data_processor.dump_async_data() + if self.config.async_dump and self.config.task in [Const.STATISTICS, Const.TENSOR]: + self.data_collector.data_processor.dump_async_data() self.data_collector.write_json() self.loop += 1 + self.currrent_step_first_debug_save = True self.reset_status() def need_stop_service(self): @@ -356,16 +374,20 @@ class Service: dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}") create_directory(dump_dir) - if self.config.task in self.data_collector.tasks_need_tensor_data: + + dump_data_dir = None + if self.config.task in self.data_collector.tasks_need_tensor_data or ( + self.config.task == Const.STATISTICS and self.config.tensor_list): dump_data_dir = os.path.join(dump_dir, "dump_tensor_data") create_directory(dump_data_dir) - else: - dump_data_dir = None dump_path_aggregation = DumpPathAggregation() - dump_path_aggregation.dump_file_path = os.path.join(dump_dir, "dump.json") - dump_path_aggregation.stack_file_path = os.path.join(dump_dir, "stack.json") - dump_path_aggregation.construct_file_path = os.path.join(dump_dir, "construct.json") + if self.config.level != Const.LEVEL_DEBUG: + dump_path_aggregation.dump_file_path = os.path.join(dump_dir, "dump.json") + dump_path_aggregation.stack_file_path = os.path.join(dump_dir, "stack.json") + dump_path_aggregation.construct_file_path = os.path.join(dump_dir, "construct.json") + else: + dump_path_aggregation.debug_file_path = os.path.join(dump_dir, "debug.json") dump_path_aggregation.dump_tensor_data_dir = dump_data_dir dump_path_aggregation.free_benchmark_file_path = os.path.join(dump_dir, "free_benchmark.csv") self.data_collector.update_dump_paths(dump_path_aggregation) @@ -378,10 +400,12 @@ class Service: functools.partial(self.build_hook, BaseScope.Module_Type_API) ) self.api_register.register_all_api() + wrap_jit_script_func() def register_module_hook(self): if self.config.level in [Const.LEVEL_L0, Const.LEVEL_MIX]: logger.info_on_rank_0(f"The module {self.config.task} hook function is successfully mounted to the model.") + ModuleProcesser.enable_module_dump = True self.module_processor.register_module_hook(self.model, self.build_hook) def attl_init(self): @@ -412,7 +436,7 @@ class Service: if self.config.nfs_path: self.attl.upload("end") elif self.attl.socket_manager is not None: - logger.info(f"pid: {os.getpid()} finished, start send STOP signal.") + logger.info(f"pid: {os.getpid()} finished, start sends STOP signal.") self.attl.socket_manager.send_stop_signal() def reset_status(self): @@ -429,36 +453,24 @@ class Service: if self.config.rank and self.current_rank not in self.config.rank: return - def init_for_debug_level(self): - if not (self.config.level == Const.LEVEL_DEBUG and self.config.task in [Const.TENSOR, Const.STATISTICS]): + def save(self, variable, name, save_backward): + if self.config.level != Const.LEVEL_DEBUG: return - try: - self.current_rank = get_rank_if_initialized() - except DistributedNotInitializedError: - self.current_rank = None - # dir: dump_path -- rank{} -- debug.json - self.dump_iter_dir = self.config.dump_path - cur_rank = self.current_rank if self.current_rank is not None else '' - dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}") - create_directory(dump_dir) - if self.config.task in self.data_collector.tasks_need_tensor_data: - dump_data_dir = os.path.join(dump_dir, "dump_tensor_data") - create_directory(dump_data_dir) - else: - dump_data_dir = None + self.current_iter = self.loop + self.init_step + if self.config.step and self.current_iter not in self.config.step: + return - dump_path_aggregation = DumpPathAggregation() - dump_path_aggregation.dump_tensor_data_dir = dump_data_dir - dump_path_aggregation.debug_file_path = os.path.join(dump_dir, "debug.json") - self.data_collector.update_dump_paths(dump_path_aggregation) - self.data_collector.initialize_json_file(framework=Const.PT_FRAMEWORK) + if self.currrent_step_first_debug_save: + try: + self.current_rank = get_rank_if_initialized() + except DistributedNotInitializedError: + self.current_rank = None - self.debug_variable_counter = defaultdict(int) + self.create_dirs() + self.debug_variable_counter = defaultdict(int) + self.currrent_step_first_debug_save = False - def save(self, variable, name, save_backward): - if self.config.level != Const.LEVEL_DEBUG: - return count = self.debug_variable_counter[name] self.debug_variable_counter[name] += 1 @@ -471,3 +483,13 @@ class Service: # backward save if save_backward: self.data_collector.debug_data_collect_backward(variable, grad_name_with_count) + + def register_custom_api(self, module, api_name, api_prefix): + self.ori_customer_func[str(module) + Const.SEP + api_name] = getattr(module, api_name) + ApiRegistry.register_custom_api(module, api_name, api_prefix, + functools.partial(self.build_hook, BaseScope.Module_Type_API), ApiTemplate) + + def restore_custom_api(self, module, api): + ori_func = self.ori_customer_func.get(str(module) + Const.SEP + api) + if ori_func: + setattr(module, api, ori_func) diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/comm_group_map/__init__.py b/debug/accuracy_tools/msprobe/test/common_set_up/__init__.py similarity index 100% rename from profiler/msprof_analyze/cluster_analyse/recipes/comm_group_map/__init__.py rename to debug/accuracy_tools/msprobe/test/common_set_up/__init__.py diff --git a/debug/accuracy_tools/msprobe/test/common_set_up/mindtorch.py b/debug/accuracy_tools/msprobe/test/common_set_up/mindtorch.py new file mode 100644 index 0000000000000000000000000000000000000000..665d17c21e743fb5ffe6a0d9e014fe0a2da4af99 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/common_set_up/mindtorch.py @@ -0,0 +1,29 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mindspore import Tensor +import torch + + +def create_msa_tensor(data, dtype=None): + return Tensor(data, dtype) + + +tensor_tensor = torch.tensor +setattr(torch, 'tensor', create_msa_tensor) + + +def reset_torch_tensor(): + setattr(torch, 'tensor', tensor_tensor) diff --git a/debug/accuracy_tools/msprobe/test/common_set_up/test_set_up.py b/debug/accuracy_tools/msprobe/test/common_set_up/test_set_up.py new file mode 100644 index 0000000000000000000000000000000000000000..de3c0272a0266f6729710c4e47dbc43fa5131f03 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/common_set_up/test_set_up.py @@ -0,0 +1,51 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +from unittest import TestCase +from unittest.mock import MagicMock + +from mindspore import mint + +try: + from mint import distributed +except ImportError: + distributed = MagicMock() + setattr(mint, 'distributed', distributed) + +from mindspore import ops +if not hasattr(ops, 'DumpGradient'): + DumpGradient = MagicMock() + setattr(ops, 'DumpGradient', DumpGradient) + +# ensure not to import torch_npu +from msprobe.mindspore import service +from msprobe.mindspore.monitor import common_func + +from .mindtorch import reset_torch_tensor +from msprobe.mindspore.common import utils +from msprobe.mindspore.common.utils import is_mindtorch + +utils.mindtorch_check_result = None +importlib.reload(service) +importlib.reload(common_func) +reset_torch_tensor() + + +class SetUp(TestCase): + def test_case(self): + self.assertTrue(hasattr(mint, 'distributed')) + self.assertTrue(is_mindtorch()) + utils.mindtorch_check_result = None diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 61766ed27c0a58f4fff81fb2f45618de60bb5b48..3cdd57752e0354a85881874200820ff49651c63e 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -47,16 +47,18 @@ from msprobe.core.common.utils import (CompareException, check_regex_prefix_format_valid, set_dump_path, get_dump_mode, - get_real_step_or_rank, - get_step_or_rank_from_string, + get_real_step_or_rank, + get_step_or_rank_from_string, get_stack_construct_by_dump_json_path, check_seed_all, safe_get_value, - recursion_depth_decorator, MsprobeBaseException, check_str_param, is_json_file, - detect_framework_by_dump_json) + detect_framework_by_dump_json, + is_save_variable_valid, + get_file_type) +from msprobe.core.common.decorator import recursion_depth_decorator class TestUtils(TestCase): @@ -206,7 +208,7 @@ class TestUtils(TestCase): with self.assertRaises(CompareException) as context: set_dump_path(input_param) self.assertEqual(context.exception.code, CompareException.INVALID_PATH_ERROR) - mock_error.assert_called_with("Please check the json path is valid. npu_path: None, bench_path: bench_path") + mock_error.assert_called_with("Please check the json path is valid and ensure that neither npu_path nor bench_path is None.") @patch.object(logger, "error") def test_get_dump_mode(self, mock_error): @@ -221,23 +223,49 @@ class TestUtils(TestCase): } input_param["npu_json_path"] = "npu_path" - with patch("msprobe.core.common.utils.load_json", return_value=npu_json): + with patch("msprobe.core.common.utils.load_json", return_value=npu_json), \ + patch("msprobe.core.common.utils.get_file_type", return_value=Const.DUMP_JSON_FILE): dump_mode = get_dump_mode(input_param) self.assertEqual(dump_mode, Const.ALL) npu_json["task"] = Const.STATISTICS with patch("msprobe.core.common.utils.load_json", return_value=npu_json), \ - patch("msprobe.core.common.utils.md5_find", return_value=True): + patch("msprobe.core.common.utils.md5_find", return_value=True), \ + patch("msprobe.core.common.utils.get_file_type", return_value=Const.DUMP_JSON_FILE): dump_mode = get_dump_mode(input_param) self.assertEqual(dump_mode, Const.MD5) npu_json["task"] = Const.OVERFLOW_CHECK - with patch("msprobe.core.common.utils.load_json", return_value=npu_json): + with patch("msprobe.core.common.utils.load_json", return_value=npu_json), \ + patch("msprobe.core.common.utils.get_file_type", return_value=Const.DUMP_JSON_FILE): with self.assertRaises(CompareException) as context: dump_mode = get_dump_mode(input_param) self.assertEqual(context.exception.code, CompareException.INVALID_TASK_ERROR) mock_error.assert_called_with("Compare applies only to task is tensor or statistics") + def test_get_file_type(self): + # 测试有效的 file_path (dump.json) + file_path = 'path/to/dump.json' + expected_file_type = Const.DUMP_JSON_FILE + self.assertEqual(get_file_type(file_path), expected_file_type) + + # 测试有效的 file_path (debug.json) + file_path = 'path/to/debug.json' + expected_file_type = Const.DEBUG_JSON_FILE + self.assertEqual(get_file_type(file_path), expected_file_type) + + # 测试无效的 file_path + file_path = 'path/to/unknown.json' + with self.assertRaises(CompareException) as context: + get_file_type(file_path) + self.assertEqual(context.exception.code, CompareException.INVALID_PATH_ERROR) + + # 测试非字符串类型的 file_path + file_path = 12345 # 非字符串类型 + with self.assertRaises(CompareException) as context: + get_file_type(file_path) + self.assertEqual(context.exception.code, CompareException.INVALID_PATH_ERROR) + @patch('msprobe.core.common.file_utils.get_file_content_bytes') def test_get_json_contents_should_raise_exception(self, mock_get_file_content_bytes): mock_get_file_content_bytes.return_value = 'not a dict' @@ -337,7 +365,7 @@ class TestUtils(TestCase): def test_recursion_depth_decorator(self, mock_error): # 测试递归深度限制函数 recursion_list = [[]] - temp_list = recursion_list[0] + temp_list = recursion_list[0] for _ in range(Const.MAX_DEPTH): temp_list.append([]) temp_list = temp_list[0] @@ -530,3 +558,40 @@ class TestDetectFrameworkByDumpJson(unittest.TestCase): result = detect_framework_by_dump_json(file_path) self.assertEqual(context.exception.code, CompareException.INVALID_PARAM_ERROR) mock_logger.error.assert_called_once_with(f"{file_path} must be based on the MindSpore or PyTorch framework.") + + +class TestIsSaveVariableValid(unittest.TestCase): + def setUp(self): + self.valid_special_types = (int, float, str, bool) + + def test_is_save_variable_valid_DepthExceeded_ReturnsFalse(self): + # 创建一个深度超过 Const.DUMP_MAX_DEPTH 的嵌套结构 + nested_structure = [0] * Const.DUMP_MAX_DEPTH + for _ in range(Const.DUMP_MAX_DEPTH): + nested_structure = [nested_structure] + self.assertFalse(is_save_variable_valid(nested_structure, self.valid_special_types)) + + def test_is_save_variable_valid_ValidSpecialTypes_ReturnsTrue(self): + for valid_type in self.valid_special_types: + self.assertTrue(is_save_variable_valid(valid_type(0), self.valid_special_types)) + + def test_is_save_variable_valid_ListWithValidElements_ReturnsTrue(self): + self.assertTrue(is_save_variable_valid([1, 2, 3], self.valid_special_types)) + + def test_is_save_variable_valid_ListWithInvalidElement_ReturnsFalse(self): + self.assertFalse(is_save_variable_valid([1, "test", [1, slice(1)]], self.valid_special_types)) + + def test_is_save_variable_valid_TupleWithValidElements_ReturnsTrue(self): + self.assertTrue(is_save_variable_valid((1, 2, 3), self.valid_special_types)) + + def test_is_save_variable_valid_TupleWithInvalidElement_ReturnsFalse(self): + self.assertFalse(is_save_variable_valid((1, "test", [1, slice(1)]), self.valid_special_types)) + + def test_is_save_variable_valid_DictWithValidElements_ReturnsTrue(self): + self.assertTrue(is_save_variable_valid({"a": 1, "b": "test"}, self.valid_special_types)) + + def test_is_save_variable_valid_DictWithInvalidKey_ReturnsFalse(self): + self.assertFalse(is_save_variable_valid({1: "test"}, self.valid_special_types)) + + def test_is_save_variable_valid_DictWithInvalidValue_ReturnsFalse(self): + self.assertFalse(is_save_variable_valid({"a": [1, slice(1)]}, self.valid_special_types)) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py index 94244be326e9954c700339abec2db16a2ab31b07..107581b5fe58446352082f0dde6c4d0e83a74246 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py @@ -6,15 +6,49 @@ import threading import unittest from unittest.mock import patch +import numpy as np import pandas as pd import torch +from msprobe.core.common.file_utils import load_json from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import Comparator, ModeConfig -from msprobe.core.compare.highlight import find_error_rows, find_compare_result_error_rows, ApiBatch -from msprobe.core.compare.utils import get_accuracy -from msprobe.pytorch.compare.pt_compare import PTComparator +from msprobe.core.compare.acc_compare import ModeConfig, MappingConfig, MappingDict, Comparator, ParseData, ProcessDf, \ + Match, CreateTable, CalcStatsDiff + +npu_op_item_data_fuzzy = { + 'op_name': 'Functional.conv2d.0.forward.input.0', + 'dtype': 'torch.float32', + 'shape': [1, 1, 28, 28], + 'summary': [3.029174327850342, -2.926689624786377, -0.06619918346405029], + 'stack_info': [], + 'data_name': 'Functional.conv2d.0.forward.input.0.pt', + 'compare_key': 'Functional.conv2d.0.forward.input.0', + 'compare_shape': [1, 1, 28, 28], +} +npu_op_item_fuzzy = pd.Series(npu_op_item_data_fuzzy) +npu_op_item_data_fuzzy_2 = { + 'op_name': 'Functional.conv2d.0.forward.input.1', + 'dtype': 'torch.float32', + 'shape': [1, 1, 28, 28], + 'summary': [3.029174327850342, -2.926689624786377, -0.06619918346405029], + 'stack_info': [], + 'data_name': 'Functional.conv2d.0.forward.input.1.pt', + 'compare_key': 'Functional.conv2d.0.forward.input.1', + 'compare_shape': [1, 1, 28, 28], +} +npu_op_item_fuzzy_2 = pd.Series(npu_op_item_data_fuzzy_2) +bench_op_item_data_fuzzy = { + 'op_name': 'Functional.conv2d.1.forward.input.0', + 'dtype': 'torch.float32', + 'shape': [1, 1, 28, 28], + 'summary': [3.029174327850342, -2.926689624786377, -0.06619918346405029], + 'stack_info': [], + 'data_name': 'Functional.conv2d.1.forward.input.0.pt', + 'compare_key': 'Functional.conv2d.1.forward.input.0', + 'compare_shape': [1, 1, 28, 28], +} +bench_op_item_fuzzy = pd.Series(bench_op_item_data_fuzzy) npu_dict = {'op_name': ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.1', 'Functional.conv2d.0.forward.input.2', 'Functional.conv2d.0.forward.output'], @@ -159,7 +193,8 @@ aten_result = [ -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Warning', 'Need double check api accuracy.', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward.output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', - ' ', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', + ' ', ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', + 'Nan', 'Yes', '', 'None'], ['Aten__native_batch_norm_legit_functional.default_0_forward.output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', ' ', ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', @@ -173,40 +208,6 @@ aten_result = [ highlight_dict = {'red_rows': [], 'yellow_rows': []} -num_0, num_1, num_2, num_3 = 0, 1, 2, 3 -summary_line_input = ['Functional_batch_norm_0_forward.input.0', 'Functional_batch_norm_0_forward.input.0', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.01, 0, 0, 0, 1, 1, 1, 1, 1.01, 1, 1, 1, - 'Yes', ''] -summary_line_1 = ['Functional_batch_norm_0_forward.output.0', 'Functional_batch_norm_0_forward.output.0', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 10, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, - 'Warning', ''] -summary_line_2 = ['Functional_batch_norm_0_forward.output.1', 'Functional_batch_norm_0_forward.output.1', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.02, 0, 0, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, - 'Warning', ''] -summary_line_3 = ['Functional_batch_norm_0_forward.output.2', 'Functional_batch_norm_0_forward.output.2', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, - 'Warning', ''] -line_input = ['Functional.batch.norm.0.forward.input.0', 'Functional.batch.norm.0.forward.input.0', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 1, 0.5, 1, 1, 0.95, 1, - 1, 1, 1, 1, 1.01, 1, 1, 1, - 'Yes', ''] -line_1 = ['Functional.batch.norm.0.forward.output.0', 'Functional.batch.norm.0.forward.output.0', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1, 1, 0.59, 1, - 'nan', 0, 1, 1, 19, 1, 1, 1, - 'Yes', ''] -line_2 = ['Functional.batch.norm.0.forward.output.1', 'Functional.batch.norm.0.forward.output.1', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.9, 0.5, 1, 1, 0.8, 1, - 0, 0.12, 0, 1, 1, 0.1, 1, 1, - 'Yes', ''] -line_3 = ['Functional.batch.norm.0.forward.output.2', 'Functional.batch.norm.0.forward.output.2', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1.1e+10, 1, 0.85, 1, - 9, 0.12, 0, 1, 1, 0.1, 1, 1, - 'Yes', ''] - op_data = { 'input_args': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, @@ -267,6 +268,33 @@ def generate_dump_json(base_dir): json.dump(data, json_file) +def generate_dump_json_md5(base_dir): + data_path = os.path.join(base_dir, 'dump_md5.json') + data = { + 'task': 'statistics', + 'level': 'L1', + 'dump_data_dir': '', + 'data': { + 'Functional.linear.0.forward': { + 'input_args': [ + {'type': 'torch.Tensor', + 'dtype': 'torch.float32', + 'shape': [2, 2], + 'Max': 2, + 'Min': 0, + 'Mean': 1, + 'Norm': 1, + 'requires_grad': False, + 'md5': 123456 + } + ] + } + } + } + with open(data_path, 'w') as json_file: + json.dump(data, json_file) + + def generate_stack_json(base_dir): data_path = os.path.join(base_dir, 'stack.json') data = {'Functional.linear.0.forward': ['File']} @@ -300,145 +328,6 @@ class TestUtilsMethods(unittest.TestCase): if os.path.exists(base_dir3): shutil.rmtree(base_dir3) - def test_get_accuracy_graph_mode(self): - result = [] - get_accuracy(result, npu_dict_aten, bench_dict_functional, dump_mode=Const.SUMMARY) - self.assertEqual(result, aten_result) - - def test_find_error_rows(self): - api_batch = ApiBatch("Functional_batch_norm_0_forward", 0) - api_batch.input_len = 1 - api_batch.output_end_index = 4 - api_batch.params_end_index = 4 - summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] - highlight_dict_test = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} - find_error_rows(summary_result, api_batch, highlight_dict_test, dump_mode=Const.SUMMARY) - self.assertEqual(highlight_dict_test, - {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []}) - - def test_find_compare_result_error_rows(self): - result = [line_input, line_1, line_2, line_3] - result_df = pd.DataFrame(result) - highlight_dict_test = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} - find_compare_result_error_rows(result_df, highlight_dict_test, dump_mode=Const.ALL) - self.assertEqual(highlight_dict_test, { - "red_rows": {1, 3}, - "yellow_rows": {2}, - "red_lines": [ - (1, ["maximum or minimum is nan, -inf, or inf"]), - (3, ["maximum absolute error exceeds 1e+10"]) - ], - "yellow_lines": [ - (2, ["The output's one thousandth err ratio decreases by more than 0.1 compared to the input/parameters's"]), - (3, [ - "maximum absolute error of both input/parameters and output exceed 1, " - "with the output larger by an order of magnitude", - "The output's cosine decreases by more than 0.1 compared to the input/parameters's"]) - ] - }) - - def test_calculate_summary_data(self): - npu_summary_data = [1, 1, 1, 1] - bench_summary_data = [2, 2, 2, 2] - result_item = ['', '', '', '', '', '', '', '', '', '', '', '', '', ''] - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - comparator = Comparator(mode_config) - comparator.calculate_summary_data(npu_summary_data, bench_summary_data, result_item) - self.assertEqual(result_item, - ['', '', '', '', '', '', -1, -1, -1, -1, '50.0%', '50.0%', '50.0%', '50.0%', '', '']) - - bench_summary_data = [0, 0, 0, 0] - result_item = ['', '', '', '', '', '', '', '', '', '', '', '', '', ''] - - comparator.calculate_summary_data(npu_summary_data, bench_summary_data, result_item) - self.assertEqual(result_item, ['', '', '', '', '', '', 1, 1, 1, 1, 'N/A', 'N/A', 'N/A', 'N/A', 'Warning', - 'Need double check api accuracy.']) - - def test_make_result_table_stack_mode_True(self): - result_md5 = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', 'File']] - result_summary = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', 'File']] - result_all = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', 'File', '-1']] - columns_md5_stack_mode_true = CompareConst.MD5_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] - result_table_md5_true = pd.DataFrame(result_md5, columns=columns_md5_stack_mode_true, dtype=object) - columns_summary_stack_mode_true = CompareConst.SUMMARY_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] - result_table_summary_true = pd.DataFrame(result_summary, columns=columns_summary_stack_mode_true, dtype=object) - columns_all_stack_mode_true = CompareConst.COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] + ['Data_name'] - result_table_all_true = pd.DataFrame(result_all, columns=columns_all_stack_mode_true, dtype=object) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - - dump_mode = Const.MD5 - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_md5) - self.assertTrue(result_df.equals(result_table_md5_true)) - - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_summary) - self.assertTrue(result_df.equals(result_table_summary_true)) - - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_all) - self.assertTrue(result_df.equals(result_table_all_true)) - - def test_make_result_table_stack_mode_False(self): - result_md5_test = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '']] - result_md5 = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '']] - result_summary_test = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', '']] - result_summary = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '']] - result_all_test = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', '', '-1']] - result_all = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', - 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', '-1']] - columns_md5_stack_mode_true = CompareConst.MD5_COMPARE_RESULT_HEADER - result_table_md5_true = pd.DataFrame(result_md5, columns=columns_md5_stack_mode_true, dtype='object') - columns_summary_stack_mode_true = CompareConst.SUMMARY_COMPARE_RESULT_HEADER - result_table_summary_true = pd.DataFrame(result_summary, columns=columns_summary_stack_mode_true, - dtype='object') - columns_all_stack_mode_true = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] - result_table_all_true = pd.DataFrame(result_all, columns=columns_all_stack_mode_true, dtype='object') - - stack_mode = False - auto_analyze = True - fuzzy_match = False - - dump_mode = Const.MD5 - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_md5_test) - self.assertTrue(result_df.equals(result_table_md5_true)) - - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_summary_test) - self.assertTrue(result_df.equals(result_table_summary_true)) - - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result_df = Comparator(mode_config).make_result_table(result_all_test) - self.assertTrue(result_df.equals(result_table_all_true)) - def test_gen_merge_list(self): op_data = { 'input_args': [ @@ -454,6 +343,7 @@ class TestUtilsMethods(unittest.TestCase): op_name = 'Functional.linear.0.forward' stack_json_data = {'Functional.linear.0.forward': ['File']} merge_list = { + 'debug_struct': [], 'input_struct': [('torch.float32', [2, 2])], 'op_name': ['Functional.linear.0.forward.input.0'], 'output_struct': [], @@ -469,32 +359,405 @@ class TestUtilsMethods(unittest.TestCase): dump_mode = Const.SUMMARY mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - result = Comparator(mode_config).gen_merge_list(json_data, op_name, stack_json_data) + result = ParseData(mode_config).gen_merge_list(json_data, op_name, stack_json_data) self.assertEqual(result, merge_list) - def test_check_op_fuzzy_false(self): + def test_check_op_item_fuzzy(self): stack_mode = False auto_analyze = True dump_mode = Const.SUMMARY - fuzzy_match = False + fuzzy_match = True mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - result = pt_comparator.check_op(npu_dict, bench_dict) + match = Match(mode_config, mapping_config, cross_frame=False) + result = match.check_op_item(npu_op_item_fuzzy, bench_op_item_fuzzy) self.assertEqual(result, True) - def test_check_op_fuzzy_true(self): - stack_mode = False + def test_compare_statistics(self): + generate_dump_json(base_dir) + generate_stack_json(base_dir) + file_list = [os.path.join(base_dir, 'dump.json'), os.path.join(base_dir, 'dump.json'), + os.path.join(base_dir, 'stack.json')] + + stack_mode = True auto_analyze = True + fuzzy_match = False dump_mode = Const.SUMMARY - - fuzzy_match = True mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - result = pt_comparator.check_op(npu_dict2, bench_dict) - self.assertEqual(result, True) + from msprobe.pytorch.compare.pt_compare import read_real_data + comparator = Comparator(read_real_data, mode_config, mapping_config) + result = comparator.compare_statistics(file_list) + o_data = [ + ['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', + 'torch.float32', 'torch.float32', '[2, 2]', '[2, 2]', 0, 0, 0, 0, '0.0%', 'N/A', '0.0%', '0.0%', + 2, 0, 1, 1, 2, 0, 1, 1, '', '', ['File'] + ] + ] + columns = CompareConst.SUMMARY_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] + o_result = pd.DataFrame(o_data, columns=columns, dtype=object) + self.assertTrue(np.array_equal(result.to_numpy(), o_result.to_numpy())) + + +class TestParseData(unittest.TestCase): + + def setUp(self): + os.makedirs(base_dir, mode=0o750, exist_ok=True) + generate_dump_json(base_dir) + generate_dump_json_md5(base_dir) + generate_stack_json(base_dir) + + self.lock = threading.Lock() + + def tearDown(self): + if os.path.exists(base_dir): + shutil.rmtree(base_dir) + + def test_parse(self): + file_list = [os.path.join(base_dir, 'dump.json'), os.path.join(base_dir, 'dump.json'), + os.path.join(base_dir, 'stack.json')] + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode) + parse_data = ParseData(mode_config) + npu_df, bench_df = parse_data.parse(file_list) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File']]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info'] + ) + self.assertTrue(npu_df.equals(target_df)) + self.assertTrue(bench_df.equals(target_df)) + + def test_gen_data_df_summary(self): + npu_json_path = os.path.join(base_dir, 'dump.json') + stack_json_path = os.path.join(base_dir, 'stack.json') + npu_json_data = load_json(npu_json_path) + stack_json_data = load_json(stack_json_path) + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode) + parse_data = ParseData(mode_config) + npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File']]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info'] + ) + self.assertTrue(npu_df.equals(target_df)) + + def test_gen_data_df_all(self): + npu_json_path = os.path.join(base_dir, 'dump.json') + stack_json_path = os.path.join(base_dir, 'stack.json') + npu_json_data = load_json(npu_json_path) + stack_json_data = load_json(stack_json_path) + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode, dump_mode=Const.ALL) + parse_data = ParseData(mode_config) + npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'Functional.linear.0.forward.input.0.pt']], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name'] + ) + self.assertTrue(npu_df.equals(target_df)) + + def test_gen_data_df_md5(self): + npu_json_path = os.path.join(base_dir, 'dump_md5.json') + stack_json_path = os.path.join(base_dir, 'stack.json') + npu_json_data = load_json(npu_json_path) + stack_json_data = load_json(stack_json_path) + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode, dump_mode=Const.MD5) + parse_data = ParseData(mode_config) + npu_df = parse_data.gen_data_df(npu_json_data, stack_json_data) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 123456]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'md5'] + ) + self.assertTrue(npu_df.equals(target_df)) + + def test_gen_merge_list(self): + npu_json_path = os.path.join(base_dir, 'dump.json') + stack_json_path = os.path.join(base_dir, 'stack.json') + npu_json_data = load_json(npu_json_path) + stack_json_data = load_json(stack_json_path) + + stack_mode = True + mode_config = ModeConfig(stack_mode=stack_mode) + parse_data = ParseData(mode_config) + merge_list = parse_data.gen_merge_list(npu_json_data, 'Functional.linear.0.forward', stack_json_data) + + target_dict = { + 'debug_struct': [], + 'input_struct': [('torch.float32', [2, 2])], + 'op_name': ['Functional.linear.0.forward.input.0'], + 'output_struct': [], + 'params_grad_struct': [], + 'params_struct': [], + 'stack_info': [['File']], + 'summary': [[2, 0, 1, 1]] + } + self.assertEqual(merge_list, target_dict) + + +class TestProcessDf(unittest.TestCase): + + def test_get_api_name_success(self): + api_list = ['Functional', 'linear', '0', 'forward', 'input', '0'] + + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + api_name = process_df.get_api_name(api_list) + + target_api_name = 'Functional.linear' + self.assertEqual(api_name, target_api_name) + + @patch('msprobe.core.compare.acc_compare.logger') + def test_get_api_name_index_error(self, mock_logger): + api_list = ['Functional'] + with self.assertRaises(CompareException) as context: + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + api_name = process_df.get_api_name(api_list) + self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) + mock_logger.error.assert_called_once_with('Failed to retrieve API name, please check if the dump data is reasonable') + + def test_process_compare_key_and_shape(self): + npu_df_o = bench_df_o = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File']]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info'] + ) + + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + npu_df, bench_df = process_df.process_compare_key_and_shape(npu_df_o, bench_df_o) + + target_df = pd.DataFrame( + [['Functional.linear.0.forward.input.0', 'torch.float32', [2, 2], [2, 0, 1, 1], ['File'], 'Functional.linear.0.forward.input.0', [2, 2]]], + columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'compare_key', 'compare_shape'] + ) + self.assertTrue(npu_df.equals(target_df)) + self.assertTrue(bench_df.equals(target_df)) + + def test_process_internal_api_mapping(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + + # mint to torch + npu_op_name = 'Mint.mean.0.input.0' + target_name = 'Torch.mean.0.input.0' + name = process_df.process_internal_api_mapping(npu_op_name) + self.assertEqual(name, target_name) + + # mintfunctional to functional + npu_op_name = 'MintFunctional.mean.0.input.0' + target_name = 'Functional.mean.0.input.0' + name = process_df.process_internal_api_mapping(npu_op_name) + self.assertEqual(name, target_name) + + # inner mapping exists + npu_op_name = 'Functional.abs.0.input.0' + mapping_dict.ms_to_pt_mapping = {'Functional.abs': 'Torch.abs'} + target_name = 'Torch.abs.0.input.0' + name = process_df.process_internal_api_mapping(npu_op_name) + self.assertEqual(name, target_name) + + # inner mapping not found + npu_op_name = 'Functional.abs.0.input.0' + mapping_dict.ms_to_pt_mapping = {} + target_name = 'Functional.abs.0.input.0' + name = process_df.process_internal_api_mapping(npu_op_name) + self.assertEqual(name, target_name) + + def test_modify_compare_data_with_user_mapping(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + mapping_dict.api_mapping_dict = [{ + 'ms_api': 'Functional.conv2d', + 'pt_api': 'Torch.conv2d', + 'ms_args': [0], + 'pt_args': [0] + }] + + npu_df = pd.DataFrame([ + ['Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0'], + ['Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0'] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'compare_key']) + bench_df = pd.DataFrame([ + ['Torch.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Torch.conv2d.0.forward.input.0'], + ['Torch.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Torch.amax.0.forward.input.0'] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'compare_key']) + + process_df.modify_compare_data_with_user_mapping(npu_df, bench_df) + + def test_get_api_indices_dict(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + + op_name_df = pd.DataFrame([ + ['Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0'], + ['Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0'] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'compare_key']) + + api_indices_dict = process_df.get_api_indices_dict(op_name_df) + expected = { + 'Functional.conv2d': [0], + 'Functional.amax': [1] + } + self.assertEqual(api_indices_dict, expected) + + def test_process_cell_mapping(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + + # not name + npu_op_name = None + name = process_df.process_cell_mapping(npu_op_name) + self.assertEqual(name, CompareConst.N_A) + + # not params_grad + npu_op_name = 'MintFunctional.embedding.0.input.0' + name = process_df.process_cell_mapping(npu_op_name) + self.assertEqual(name, CompareConst.N_A) + + # default replace + npu_op_name = 'Cell.network_with_loss.module.GPTModel.forward.1.input.0' + name = process_df.process_cell_mapping(npu_op_name) + self.assertEqual(name, 'Module.network_with_loss.module.GPTModel.forward.1.input.0') + + # mapping_dict + npu_op_name = 'Cell.fc1.Dense.forward.0.input.0' + mapping_dict.cell_mapping_dict = {'fc1.Dense': 'module.name'} + name = process_df.process_cell_mapping(npu_op_name) + self.assertEqual(name, 'Module.module.name.forward.0.input.0') + + def test_process_data_mapping(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + mapping_dict = MappingDict(mapping_config) + process_df = ProcessDf(mode_config, mapping_config, mapping_dict) + + npu_op_name = 'Functional.flash_attention_score.4.forward.input.0' + mapping_dict.data_mapping_dict = {'Functional.flash_attention_score.4.forward.input.0': 'NPU.npu_fusion_attention.4.forward.input.0'} + name = process_df.process_data_mapping(npu_op_name) + self.assertEqual(name, 'NPU.npu_fusion_attention.4.forward.input.0') + + +class TestMatch(unittest.TestCase): + + def test_put_unmatched_in_table(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + match_result = pd.DataFrame(columns=CompareConst.MATCH_RESULT_COLUMNS) + npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], + index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + 'compare_key', 'compare_shape'] + ) + match_result = match.put_unmatched_in_table(match_result, npu_op_item) + target_match_result = pd.DataFrame([['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2], + 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A']], + columns=CompareConst.MATCH_RESULT_COLUMNS) + self.assertTrue(match_result.equals(target_match_result)) + + def test_put_matched_in_table(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + match_result = pd.DataFrame(columns=CompareConst.MATCH_RESULT_COLUMNS) + npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], + index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + 'compare_key', 'compare_shape'] + ) + bench_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2]], + index=['op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y', + 'compare_key', 'compare_shape'] + ) + match_result = match.put_matched_in_table(match_result, npu_op_item, bench_op_item) + target_match_result = pd.DataFrame([['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'op', [1, 2], + 'op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name']], + columns=CompareConst.MATCH_RESULT_COLUMNS) + self.assertTrue(match_result.equals(target_match_result)) + + def test_rename_api(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + op_name_1 = 'Functional.linear.0.forward.input.0' + result_1 = match.rename_api(op_name_1) + self.assertTrue(result_1, 'Functional.linear.input.0') + + op_name_2 = 'Functional.linear.0.backward.input.0' + result_2 = match.rename_api(op_name_2) + self.assertTrue(result_2, 'Functional.linear.input.0') + + op_name_3 = 'Functional.linear.0.x.input.0' + result_3 = match.rename_api(op_name_3) + self.assertTrue(result_3, 'Functional.linear.0.x.input.0') + + def test_check_op_item(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + npu_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'Functional.linear.0.forward.input.0', [1, 2]], + index=['op_name_x', 'dtype_x', 'shape_x', 'summary_x', 'stack_info_x', 'data_name_x', + 'compare_key', 'compare_shape'] + ) + bench_op_item = pd.Series(['op', 'float32', [1, 2], 'summary', 'stack_info', 'data_name', 'Functional.linear.1.forward.input.0', [1, 2]], + index=['op_name_y', 'dtype_y', 'shape_y', 'summary_y', 'stack_info_y', 'data_name_y', + 'compare_key', 'compare_shape'] + ) + result = match.check_op_item(npu_op_item, bench_op_item) + self.assertTrue(result) + + def test_process_fuzzy_match(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=False) + + npu_df = pd.DataFrame([ + ['Functional.conv2d.3.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.3.forward.input.0.pt', 'Functional.conv2d.3.forward.input.0', [1, 2]], + ['Functional.amax.1.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.1.forward.input.0', [1, 2]] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name', 'compare_key', 'compare_shape']) + bench_df = pd.DataFrame([ + ['Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0.pt', 'Functional.conv2d.0.forward.input.0', [1, 2]], + ['Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.0.forward.input.0', [1, 2]] + ], columns=['op_name', 'dtype', 'shape', 'summary', 'stack_info', 'data_name', 'compare_key', 'compare_shape']) + + match_result = match.process_fuzzy_match(npu_df, bench_df) + expected = pd.DataFrame( + [ + ['Functional.conv2d.3.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.3.forward.input.0.pt', 'Functional.conv2d.3.forward.input.0', [1, 2], 'Functional.conv2d.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.conv2d.0.forward.input.0.pt'], + ['Functional.amax.1.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt', 'Functional.amax.1.forward.input.0', [1, 2], 'Functional.amax.0.forward.input.0', 'float32', [1, 2], 'summary', 'stack_info', 'Functional.amax.0.forward.input.0.pt'] + ] + , columns=CompareConst.MATCH_RESULT_COLUMNS) + + self.assertTrue(match_result.equals(expected)) def test_match_op_both_last_element(self): stack_mode = False @@ -502,9 +765,10 @@ class TestUtilsMethods(unittest.TestCase): fuzzy_match = False dump_mode = Const.SUMMARY mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - a, b = pt_comparator.match_op([npu_dict], [bench_dict]) + match = Match(mode_config, mapping_config, cross_frame=False) + a, b = match.match_op([npu_op_item_fuzzy], [bench_op_item_fuzzy]) self.assertEqual(a, 0) self.assertEqual(b, 0) @@ -514,9 +778,10 @@ class TestUtilsMethods(unittest.TestCase): fuzzy_match = False dump_mode = Const.SUMMARY mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - a, b = pt_comparator.match_op([npu_dict], [bench_dict, 1]) + match = Match(mode_config, mapping_config, cross_frame=False) + a, b = match.match_op([npu_op_item_fuzzy], [bench_op_item_fuzzy, 1]) self.assertEqual(a, 0) self.assertEqual(b, 0) @@ -526,217 +791,102 @@ class TestUtilsMethods(unittest.TestCase): fuzzy_match = False dump_mode = Const.SUMMARY mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() - pt_comparator = PTComparator(mode_config) - a, b = pt_comparator.match_op([npu_dict, npu_dict2], [bench_dict]) + match = Match(mode_config, mapping_config, cross_frame=False) + a, b = match.match_op([npu_op_item_fuzzy, npu_op_item_data_fuzzy_2], [bench_op_item_fuzzy]) self.assertEqual(a, 0) self.assertEqual(b, 0) - def test_compare_process(self): - generate_dump_json(base_dir) - generate_stack_json(base_dir) - file_lists = [os.path.join(base_dir, 'dump.json'), os.path.join(base_dir, 'dump.json'), - os.path.join(base_dir, 'stack.json')] + def test_gen_dtype_condition(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=True) - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + # data mapping + mapping_config.data_mapping = True + match_result = pd.DataFrame([1, 2, 3]) + result = match.gen_dtype_condition(match_result) + expected = pd.Series([True, True, True]) + self.assertTrue(result.equals(expected)) - result = PTComparator(mode_config).compare_process(file_lists) - o_data = [ - ['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], 0, 0, 0, 0, '0.0%', 'N/A', '0.0%', '0.0%', - 2, 0, 1, 1, 2, 0, 1, 1, '', '', ['File'] - ] - ] - columns = CompareConst.SUMMARY_COMPARE_RESULT_HEADER + ['NPU_Stack_Info'] - o_result = pd.DataFrame(o_data, columns=columns, dtype=object) - self.assertTrue(result.equals(o_result)) + # normal + mapping_config.data_mapping = None + match_result = pd.DataFrame([['Float16', 'Float32'], ['torch.float32', 'torch.bfloat16']], columns=['dtype_x', 'dtype_y']) + result = match.gen_dtype_condition(match_result) + expected = pd.Series([True, True]) + self.assertTrue(result.equals(expected)) - def test_merge_data(self): - op_data = { - 'input_args': [ - { - 'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [2, 2], - 'Max': 1, 'Min': 1, 'Mean': 1, 'Norm': 1, 'requires_grad': False, - 'data_name': 'Functional.linear.0.forward.input.0.pt', - 'full_op_name': 'Functional.linear.0.forward.input.0' - } - ] - } - json_data = {'data': {'Functional.linear.0.forward': op_data}} - stack_json_data = {'Functional.linear.0.forward': ['File']} - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - result = Comparator(mode_config).merge_data(json_data, stack_json_data) - ops_all = { - 'Functional.linear.0.forward.input.0': { - 'data_name': None, 'stack_info': [['File']], - 'struct': ('torch.float32', [2, 2]), 'summary': [1, 1, 1, 1] - } - } - self.assertEqual(result, ops_all) - - def test_compare_core_basic(self): - generate_dump_json(base_dir2) - generate_stack_json(base_dir2) - input_params = { - "npu_json_path": os.path.join(base_dir2, "dump.json"), - "bench_json_path": os.path.join(base_dir2, "dump.json"), - "stack_json_path": os.path.join(base_dir2, "stack.json"), - } - output_path = base_dir2 - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - PTComparator(mode_config).compare_core(input_params, output_path) - - output_files = os.listdir(output_path) - self.assertTrue(any(f.endswith(".xlsx") for f in output_files)) - - def test_compare_ops(self): - generate_dump_json(base_dir3) - generate_stack_json(base_dir3) - generate_pt(pt_dir) - dump_path = os.path.join(base_dir3, 'dump.json') - stack_path = os.path.join(base_dir3, 'stack.json') - input_param = {'npu_json_path': dump_path, 'bench_json_path': dump_path, 'stack_json_path': stack_path, - 'is_print_compare_log': True, 'npu_dump_data_dir': pt_dir, 'bench_dump_data_dir': pt_dir} - dump_path_dict = {'Functional.linear.0.forward.input.0': ['Functional.linear.0.forward.input.0.pt', - 'Functional.linear.0.forward.input.0.pt']} - result_df = pd.DataFrame({ - 'NPU Name': ['Functional.linear.0.forward.input.0'], - 'Bench Name': ['Functional.linear.0.forward.input.0'] - }) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - pt_comparator = PTComparator(mode_config) - updated_df = pt_comparator.compare_ops(idx=0, dump_path_dict=dump_path_dict, result_df=result_df, - lock=self.lock, input_param=input_param) - - self.assertEqual(updated_df.loc[0, CompareConst.COSINE], 1.0) - self.assertEqual(updated_df.loc[0, CompareConst.MAX_ABS_ERR], 0) - - def test_do_multi_process(self): - data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], - '', '', '', '', '', '', 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', ['-1', '-1']]] - o_data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', - 'torch.float32', 'torch.float32', [2, 2], [2, 2], - 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 1, 1, 1, 1, 1, 1, 1, 1, 'None', 'No bench data matched.', ['-1', '-1']]] - columns = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] - result_df = pd.DataFrame(data, columns=columns) - o_result = pd.DataFrame(o_data, columns=columns) - generate_dump_json(base_dir) - input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} + def test_process_cross_frame_dtype(self): + mode_config = ModeConfig() + mapping_config = MappingConfig() + match = Match(mode_config, mapping_config, cross_frame=True) - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + dtype_o = pd.Series(['Int8', 'Float16', 'torch.bool', 'Complex64', 'unknown']) + dtype = match.process_cross_frame_dtype(dtype_o) + self.assertTrue(dtype.equals(pd.Series(['int', 'float', 'bool', 'complex', 'unknown']))) - comparator = Comparator(mode_config) - result = comparator.do_multi_process(input_param, result_df) - self.assertTrue(result.equals(o_result)) - def test_compare_by_op_1(self): - npu_op_name = 'Functional.linear.0.forward.input.0' - bench_op_name = 'N/A' - op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [-1, -1]} - input_param = {} +class TestCreateTable(unittest.TestCase): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + def test_process_data_name(self): + mode_config = ModeConfig() + create_table = CreateTable(mode_config) - pt_comparator = PTComparator(mode_config) - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + data = { + 'data_name_x': ['A', 'B', 'C'], + 'data_name_y': ['X', 'Y', 'Z'] + } + result_o = pd.DataFrame(data) + result = create_table.process_data_name(result_o) + target_data = { + 'data_name_x': [['A', 'X'], ['B', 'Y'], ['C', 'Z']], + 'data_name_y': ['X', 'Y', 'Z'] + } + target_result = pd.DataFrame(target_data) + self.assertTrue(result.equals(target_result)) - self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', 'No bench data matched.']) + def test_set_summary(self): + mode_config = ModeConfig() + create_table = CreateTable(mode_config) - def test_compare_by_op_2(self): - npu_op_name = 'Functional.linear.0.forward.input.0' - bench_op_name = 'Functional.linear.0.forward.input.0' + # all nan + result = create_table.set_summary(['nan', 'NaN', 'nAn']) + expected = [CompareConst.NAN, CompareConst.NAN, CompareConst.NAN] + self.assertEqual(result, expected) - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + # mixed values + result = create_table.set_summary([1, 'nan', 2.0, 'NaN']) + expected = [1, CompareConst.NAN, 2.0, CompareConst.NAN] + self.assertEqual(result, expected) - pt_comparator = PTComparator(mode_config) + # NA case + result = create_table.set_summary(CompareConst.N_A) + expected = [CompareConst.N_A, CompareConst.N_A, CompareConst.N_A, CompareConst.N_A] + self.assertEqual(result, expected) - pt_name = '-1' - op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} - input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) - self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', 'No bench data matched.']) + # empty input + result = create_table.set_summary([]) + expected = [] + self.assertEqual(result, expected) - pt_name = 'Functional.linear.0.forward.input.0.pt' - op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} - input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) - self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', 'Dump file: Functional.linear.0.forward.input.0.pt not found.']) - generate_pt(base_dir) - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) - self.assertEqual(result, [1.0, 0.0, 0.0, 0.0, 1.0, 1.0, '']) +class TestCalcStatsDiff(unittest.TestCase): + def test_type_check(self): + mode_config = ModeConfig() + calc_stats_diff = CalcStatsDiff(mode_config) -class TestComparator(unittest.TestCase): - def setUp(self): - mode_config = ModeConfig(dump_mode=Const.MD5) - self.comparator = Comparator(mode_config=mode_config) - self.npu_ops_all = { - 'op1': {'struct': ['float32', [1, 96, 2], '83dcefb7']}, - } - self.bench_ops_all = { - 'op1': {'struct': ['float32', [1, 96, 2], '83dcefb7']}, - } + series = pd.Series([float('nan'), 5, 'nan', 10, 'abc', None]) + result = calc_stats_diff.type_check(series) + expected = pd.Series([True, True, True, True, False, False]) + self.assertTrue(result.equals(expected)) - def test_normal(self): - expected_result = ['op1', 'op1', 'float32', 'float32', [1, 96, 2], [1, 96, 2], '83dcefb7', '83dcefb7', - CompareConst.PASS, CompareConst.NONE] - result = self.comparator.get_result_md5_compare('op1', 'op1', - self.npu_ops_all, self.bench_ops_all) - self.assertEqual(result, expected_result) + def test_get_number(self): + mode_config = ModeConfig() + calc_stats_diff = CalcStatsDiff(mode_config) - @patch('msprobe.core.compare.acc_compare.logger') - def test_length_exception(self, mock_logger): - self.npu_ops_all['op1']['struct'] = ['npu_val1', 'npu_val2'] - with self.assertRaises(CompareException) as context: - self.comparator.get_result_md5_compare('op1', 'op1', - self.npu_ops_all, self.bench_ops_all) - self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - mock_logger.error.assert_called_once_with("The length of npu_struct and bench_struct must be >= 3, " - "but got npu_struct=2 and bench_struct=3. Please check!") - - def test_with_extra_args(self): - expected_result = ['op1', 'op1', 'float32', 'float32', [1, 96, 2], [1, 96, 2], '83dcefb7', '83dcefb7', - CompareConst.PASS, 'extra_data'] - result = self.comparator.get_result_md5_compare('op1', 'op1', - self.npu_ops_all, self.bench_ops_all, True, ['extra_data']) - self.assertEqual(result, expected_result) + series = pd.Series([1, '2', 3.5, 'text', None]) + result = calc_stats_diff.get_number(series) + expected = pd.Series([1, 2, 3.5, float('nan'), float('nan')]) + self.assertTrue(result.equals(expected)) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py index a1e5f8eee1bce9b170e6f4f7fdfeda65d47252c9..1a0a33f799724ffefe73bf8f024e0146b2925464 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_check.py @@ -1,7 +1,6 @@ # coding=utf-8 import unittest -from msprobe.core.compare.check import check_struct_match, check_type_shape_match, check_graph_mode, fuzzy_check_op, \ - fuzzy_check_name, check_dump_json_str, check_json_key_value, valid_key_value, check_stack_json_str +from msprobe.core.compare.check import check_dump_json_str, check_json_key_value, valid_key_value, check_stack_json_str from msprobe.core.common.utils import CompareException @@ -65,87 +64,6 @@ op_name = 'Functional.conv2d.0.backward.input.0' class TestUtilsMethods(unittest.TestCase): - - def test_check_struct_match_success(self): - result = check_struct_match(npu_dict, bench_dict) - self.assertTrue(result) - - def test_check_struct_match_fail(self): - npu_dict2 = {'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), - ('torch.float32', [16])], - 'output_struct': [('torch.float32', [1, 16, 28, 28])] - } - - bench_dict2 = {'input_struct': [('torch.float32', [2, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), - ('torch.float32', [16])], - 'output_struct': [('torch.float32', [1, 16, 28, 28])] - } - result = check_struct_match(npu_dict2, bench_dict2) - self.assertFalse(result) - - def test_check_struct_index_error(self): - npu_dict3 = {'input_struct': [('a'), ('torch.float32'), - ('torch.float32')], - 'output_struct': [('torch.float32')] - } - - bench_dict3 = {'input_struct': [('torch.float32'), ('torch.float32'), - ('torch.float32')], - 'output_struct': [('torch.float32')] - } - with self.assertRaises(CompareException) as context: - result = check_struct_match(npu_dict3, bench_dict3) - self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - - def test_check_type_shape_match_success(self): - result = check_type_shape_match(npu_struct, bench_struct) - self.assertTrue(result) - - def test_check_type_shape_match_index_error(self): - npu_struct2 = [('a'), ('torch.float32'), ('torch.float32')] - bench_struct2 = [('torch.float32'), ('torch.float32'), ('torch.float32')] - with self.assertRaises(CompareException) as context: - result = check_type_shape_match(npu_struct2, bench_struct2) - self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - - def test_check_graph_mode(self): - op1 = "Aten" - op2 = "torch" - self.assertTrue(check_graph_mode(op1, op2)) - self.assertTrue(check_graph_mode(op2, op1)) - self.assertFalse(check_graph_mode(op1, op1)) - self.assertFalse(check_graph_mode(op2, op2)) - - def test_fuzzy_check_op_1(self): - npu_name_list = [] - bench_name_list = [] - result = fuzzy_check_op(npu_name_list, bench_name_list) - self.assertFalse(result) - - def test_fuzzy_check_op_2(self): - npu_name_list = [] - bench_name_list = ['Functional.conv2d.0.forward.input.0'] - result = fuzzy_check_op(npu_name_list, bench_name_list) - self.assertFalse(result) - - def test_fuzzy_check_op_3(self): - npu_name_list = ['Functional.conv2d.0.forward.input.0'] - bench_name_list = ['Functional.conv2d.1.forward.input.0'] - result = fuzzy_check_op(npu_name_list, bench_name_list) - self.assertTrue(result) - - def test_fuzzy_check_name_1(self): - npu_name = 'Functional.conv2d.0.backward.input.0' - bench_name = 'Functional.conv2d.1.backward.input.0' - result = fuzzy_check_name(npu_name, bench_name) - self.assertTrue(result) - - def test_fuzzy_check_name_2(self): - npu_name = 'Functional.conv2d.0.backward.input.0' - bench_name = 'Functional.conv2d.1.backward.input.1' - result = fuzzy_check_name(npu_name, bench_name) - self.assertFalse(result) - def test_check_dump_json_str(self): with self.assertRaises(CompareException) as context: check_dump_json_str(op_data, op_name) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_npy_compare.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_npy_compare.py index da315b657c8c1fc691136a1dbc56574d69c92076..a30d693f7b32a806dee8667e42794259e7785545 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_npy_compare.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_npy_compare.py @@ -454,7 +454,7 @@ class TestUtilsMethods(unittest.TestCase): result, err_msg = error_value_process(n_value) - self.assertEqual(result, 0) + self.assertEqual(result, CompareConst.UNSUPPORTED) self.assertEqual(err_msg, "") def test_error_value_process_shape_unmatch(self): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index bf23f4de1dac73a44a2497e1a927ba30e5440715..4a7c421a1b84dc4b71e258beb52761f35a44aa3d 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -12,9 +12,8 @@ import numpy as np from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.utils import CompareException from msprobe.core.compare.utils import ApiItemInfo, _compare_parser, check_and_return_dir_contents, extract_json, \ - count_struct, get_accuracy, append_stack_info, get_rela_diff_summary_mode, get_un_match_accuracy, merge_tensor, \ - op_item_parse, read_op, rename_api, resolve_api_special_parameters, result_item_init, stack_column_process, \ - table_value_is_valid, get_name_and_state, reorder_op_name_list, reorder_op_x_list, gen_op_item + count_struct, get_accuracy, get_rela_diff_summary_mode, merge_tensor, op_item_parse, read_op, result_item_init, \ + stack_column_process, table_value_is_valid, get_name_and_state, reorder_op_name_list, reorder_op_x_list, gen_op_item # test_read_op_1 op_data = { @@ -272,6 +271,7 @@ result_op_dict = {'op_name': ['Tensor.add_.0.forward.input.0', 'Tensor.add_.0.fo 'output_struct': [('torch.float32', [16, 1, 3, 3])], 'params_struct': [], 'params_grad_struct': [], + 'debug_struct': [], 'summary': [[0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275], [0.003992878366261721, -0.008102823048830032, -0.0002002553956117481, 0.02844562754034996], @@ -295,6 +295,7 @@ result_op_dict_md5 = {'op_name': ['Tensor.add_.0.forward.input.0', 'Tensor.add_. 'output_struct': [('torch.float32', [16, 1, 3, 3], 2)], 'params_struct': [], 'params_grad_struct': [], + 'debug_struct': [], 'summary': [ [0.003992878366261721, -0.008102823048830032, -0.0002002553956117481, 0.02844562754034996], [-0.1, -0.1, -0.1, -0.1], @@ -306,7 +307,7 @@ base_dir2 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_acc_ def create_json_files(base_dir): - file_names = ['dump.json', 'stack.json', 'construct.json'] + file_names = ['dump.json', 'stack.json', 'construct.json', 'debug.json'] for file_name in file_names: file_path = os.path.join(base_dir, file_name) @@ -339,29 +340,20 @@ class TestUtilsMethods(unittest.TestCase): def test_extract_json_1(self): create_json_files(base_dir1) - result = extract_json(base_dir1, stack_json=False) + result = extract_json(base_dir1, Const.DUMP_JSON_FILE) self.assertEqual(result, os.path.join(base_dir1, 'dump.json')) - result = extract_json(base_dir1, stack_json=True) + result = extract_json(base_dir1, Const.STACK_JSON_FILE) self.assertEqual(result, os.path.join(base_dir1, 'stack.json')) + result = extract_json(base_dir1, Const.DEBUG_JSON_FILE) + self.assertEqual(result, os.path.join(base_dir1, 'debug.json')) + def test_check_and_return_dir_contents(self): create_rank_dirs(base_dir2) result = check_and_return_dir_contents(base_dir2, 'rank') self.assertEqual(set(result), set(['rank0', 'rank1'])) - def test_rename_api_1(self): - test_name_1 = "Distributed.broadcast.0.forward.input.0" - expect_name_1 = "Distributed.broadcast.input.0" - actual_name_1 = rename_api(test_name_1, "forward") - self.assertEqual(actual_name_1, expect_name_1) - - def test_rename_api_2(self): - test_name_2 = "Torch.sum.0.backward.output.0" - expect_name_2 = "Torch.sum.output.0" - actual_name_2 = rename_api(test_name_2, "backward") - self.assertEqual(actual_name_2, expect_name_2) - def test_read_op(self): result = read_op(op_data, op_name) self.assertEqual(result, op_result) @@ -379,11 +371,6 @@ class TestUtilsMethods(unittest.TestCase): op_item_parse(parse_item, parse_op_name, depth=11) self.assertEqual(context.exception.code, CompareException.RECURSION_LIMIT_ERROR) - def test_resolve_api_special_parameters(self): - item_list = [] - resolve_api_special_parameters(data_dict, full_op_name, item_list) - self.assertEqual(item_list, o_result_api_special) - def test_get_rela_diff_summary_mode_float_or_int(self): result_item = [0] * 14 err_msg = '' @@ -449,57 +436,6 @@ class TestUtilsMethods(unittest.TestCase): get_accuracy(result, npu_dict, bench_dict, dump_mode=Const.SUMMARY) self.assertEqual(result, o_result) - def test_append_stack_info_stack_exist_index_0(self): - result_item = ['item1'] - npu_stack_info = ['stack_info1'] - index = 0 - - append_stack_info(result_item, npu_stack_info, index) - - self.assertEqual(result_item, ['item1', 'stack_info1']) - - def test_append_stack_info_stack_exist_index_not_0(self): - result_item = ['item1'] - npu_stack_info = ['stack_info1'] - index = 1 - - append_stack_info(result_item, npu_stack_info, index) - - self.assertEqual(result_item, ['item1', CompareConst.NONE]) - - def test_append_stack_info_stack_empty_index_0(self): - result_item = ['item1'] - npu_stack_info = [] - index = 0 - - append_stack_info(result_item, npu_stack_info, index) - - self.assertEqual(result_item, ['item1', CompareConst.NONE]) - - def test_append_stack_info_stack_empty_index_not_0(self): - result_item = ['item1'] - npu_stack_info = [] - index = 1 - - append_stack_info(result_item, npu_stack_info, index) - - self.assertEqual(result_item, ['item1', CompareConst.NONE]) - - def test_get_un_match_accuracy_md5(self): - result = [] - get_un_match_accuracy(result, npu_dict, dump_mode=Const.MD5) - self.assertEqual(result, o_result_unmatch_1) - - def test_get_un_match_accuracy_summary(self): - result = [] - get_un_match_accuracy(result, npu_dict, dump_mode=Const.SUMMARY) - self.assertEqual(result, o_result_unmatch_2) - - def test_get_un_match_accuracy_all(self): - result = [] - get_un_match_accuracy(result, npu_dict, dump_mode=Const.ALL) - self.assertEqual(result, o_result_unmatch_3) - def test_merge_tensor_summary(self): op_dict = merge_tensor(tensor_list, dump_mode=Const.SUMMARY) self.assertEqual(op_dict, result_op_dict) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py index 3261bce5d6d0a15d8e46c7d9fc22df0cf64c9e4d..5ffc0013fad8cfa289a79f5aaf39219b31b77c07 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_highlight.py @@ -12,12 +12,44 @@ import openpyxl from openpyxl import load_workbook from openpyxl.styles import PatternFill - from msprobe.core.common.const import CompareConst, Const from msprobe.core.compare.highlight import ApiBatch, CheckMaxRelativeDiff, CheckOrderMagnitude, \ - CheckOneThousandErrorRatio, CheckCosineSimilarity, add_highlight_row_info, compare_result_df_convert, \ - df_malicious_value_check, find_error_rows, highlight_rows_xlsx, update_highlight_err_msg, value_check - + CheckOneThousandErrorRatio, CheckCosineSimilarity, add_highlight_row_info, HighLight +from msprobe.core.compare.config import ModeConfig + + +summary_line_input = ['Functional_batch_norm_0_forward.input.0', 'Functional_batch_norm_0_forward.input.0', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.01, 0, 0, 0, 1, 1, 1, 1, 1.01, 1, 1, 1, + 'Yes', ''] +summary_line_1 = ['Functional_batch_norm_0_forward.output.0', 'Functional_batch_norm_0_forward.output.0', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 10, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, + 'Warning', ''] +summary_line_2 = ['Functional_batch_norm_0_forward.output.1', 'Functional_batch_norm_0_forward.output.1', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.02, 0, 0, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, + 'Warning', ''] +summary_line_3 = ['Functional_batch_norm_0_forward.output.2', 'Functional_batch_norm_0_forward.output.2', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, + 'Warning', ''] +line_input = ['Functional.batch.norm.0.forward.input.0', 'Functional.batch.norm.0.forward.input.0', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 1, 0.5, 1, 1, 0.95, 1, + 1, 1, 1, 1, 1.01, 1, 1, 1, + 'Yes', ''] +line_1 = ['Functional.batch.norm.0.forward.output.0', 'Functional.batch.norm.0.forward.output.0', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1, 1, 0.59, 1, + 'nan', 0, 1, 1, 19, 1, 1, 1, + 'Yes', ''] +line_2 = ['Functional.batch.norm.0.forward.output.1', 'Functional.batch.norm.0.forward.output.1', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.9, 0.5, 1, 1, 0.8, 1, + 0, 0.12, 0, 1, 1, 0.1, 1, 1, + 'Yes', ''] +line_3 = ['Functional.batch.norm.0.forward.output.2', 'Functional.batch.norm.0.forward.output.2', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 0.5, 1.1e+10, 1, 0.85, 1, + 9, 0.12, 0, 1, 1, 0.1, 1, 1, + 'Yes', ''] base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_highlight') @@ -161,7 +193,7 @@ class TestUtilsMethods(unittest.TestCase): num = 1 info = (api_in, api_out, num) CheckMaxRelativeDiff().apply(info, color_columns, dump_mode=Const.SUMMARY) - red_lines, yellow_lines = [], [(1, ["The output's maximum relative error exceeds 0.1, while the input/parameters's is below 0.01"])] + red_lines, yellow_lines = [], [(1, ["The output's maximum relative error exceeds 0.1, while the input/parameter's is below 0.01"])] target_color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) self.assertEqual(color_columns, target_color_columns) @@ -178,45 +210,6 @@ class TestUtilsMethods(unittest.TestCase): result = CheckMaxRelativeDiff().apply(info, color_columns, dump_mode=Const.SUMMARY) self.assertEqual(result, None) - def test_find_error_rows_normal(self): - compare_result = np.array([ - ["Functional.linear.0.forward.input.0", "Functional.linear.0.forward.input.0", - "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", - 1, 1, 1, 1, 1, 1, 1, 1, "", ""], - ["Functional.linear.0.forward.input.1", "Functional.linear.0.forward.input.1", - "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", - 1, 1, 1, 1, 1, 1, 1, 1, "", ""], - ["Functional.linear.0.forward.input.2", "Functional.linear.0.forward.input.2", - "torch.float32", "torch.float32", [2], [2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", - 1, 1, 1, 1, 1, 1, 1, 1, "", ""], - ["Functional.linear.0.forward.output.0", "Functional.linear.0.forward.output.0", - "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", - 1, 1, 1, 1, 1, 1, 1, 1, "", ""], - ], dtype=object) - api_batch = ApiBatch("Functional.linear.0.forward", 0) - api_batch.input_len = 3 - api_batch.output_end_index = 4 - api_batch.params_end_index = 4 - highlight_dict = {"red_lines": [], "red_rows": set(), "yellow_lines": [], "yellow_rows": set()} - dump_mode = Const.ALL - - find_error_rows(compare_result, api_batch, highlight_dict, dump_mode) - - self.assertEqual(highlight_dict, {"red_lines": [], "red_rows": set(), "yellow_lines": [], "yellow_rows": set()}) - - def test_find_error_rows_md5(self): - compare_result = [] - api_batch = ApiBatch("", 0) - api_batch.input_len = 0 - api_batch.output_end_index = 1 - api_batch.params_end_index = 1 - highlight_dict = {} - dump_mode = Const.MD5 - - result = find_error_rows(compare_result, api_batch, highlight_dict, dump_mode) - - self.assertEqual(result, None) - def test_ApiBatch_increment_input(self): api_name = "functional.conv2d" start = 2 @@ -297,6 +290,48 @@ class TestUtilsMethods(unittest.TestCase): self.assertEqual(api_batch.output_end_index, 5) self.assertEqual(api_batch.params_grad_end_index, 5) + + def test_find_error_rows_normal(self): + compare_result = np.array([ + ["Functional.linear.0.forward.input.0", "Functional.linear.0.forward.input.0", + "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", + 1, 1, 1, 1, 1, 1, 1, 1, "", ""], + ["Functional.linear.0.forward.input.1", "Functional.linear.0.forward.input.1", + "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", + 1, 1, 1, 1, 1, 1, 1, 1, "", ""], + ["Functional.linear.0.forward.input.2", "Functional.linear.0.forward.input.2", + "torch.float32", "torch.float32", [2], [2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", + 1, 1, 1, 1, 1, 1, 1, 1, "", ""], + ["Functional.linear.0.forward.output.0", "Functional.linear.0.forward.output.0", + "torch.float32", "torch.float32", [2, 2], [2, 2], 0.0, 0.0, 0.0, 0.0, "0.0%", "0.0%", "0.0%", "0.0%", + 1, 1, 1, 1, 1, 1, 1, 1, "", ""], + ], dtype=object) + api_batch = ApiBatch("Functional.linear.0.forward", 0) + api_batch.input_len = 3 + api_batch.output_end_index = 4 + api_batch.params_end_index = 4 + highlight_dict = {"red_lines": [], "red_rows": set(), "yellow_lines": [], "yellow_rows": set()} + + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.find_error_rows(compare_result, api_batch, highlight_dict) + + self.assertEqual(highlight_dict, {"red_lines": [], "red_rows": set(), "yellow_lines": [], "yellow_rows": set()}) + + def test_find_error_rows_md5(self): + compare_result = [] + api_batch = ApiBatch("", 0) + api_batch.input_len = 0 + api_batch.output_end_index = 1 + api_batch.params_end_index = 1 + highlight_dict = {} + + mode_config = ModeConfig(dump_mode=Const.MD5) + highlight = HighLight(mode_config) + result = highlight.find_error_rows(compare_result, api_batch, highlight_dict) + + self.assertEqual(result, None) + @patch("msprobe.core.compare.highlight.logger") def test_value_check(self, mock_logger): value = "=functional.conv2d" @@ -304,7 +339,9 @@ class TestUtilsMethods(unittest.TestCase): i = 1 result_df_columns = CompareConst.COMPARE_RESULT_HEADER - value_check(value, api_name, i, result_df_columns) + mode_config = ModeConfig() + highlight = HighLight(mode_config) + highlight.value_check(value, api_name, i, result_df_columns) mock_logger.error.assert_called_once_with( "Malicious value [=functional.conv2d] at api_name [=functional.conv2d], column [Bench Name], " @@ -319,11 +356,15 @@ class TestUtilsMethods(unittest.TestCase): ] result_df = pd.DataFrame(data, columns=columns) - df_malicious_value_check(result_df, columns) + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.df_malicious_value_check(result_df, columns) def test_compare_result_df_convert(self): value = float("nan") - result = compare_result_df_convert(value) + mode_config = ModeConfig() + highlight = HighLight(mode_config) + result = highlight.compare_result_df_convert(value) self.assertEqual(result, "nan\t") def test_highlight_rows_xlsx_red(self): @@ -335,7 +376,11 @@ class TestUtilsMethods(unittest.TestCase): result_df = pd.DataFrame(data, columns=columns) highlight_dict = {'red_rows': [0]} file_path = os.path.join(base_dir, 'result.xlsx') - highlight_rows_xlsx(result_df, highlight_dict, file_path) + + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) + generate_result_xlsx(base_dir) self.assertTrue(compare_excel_files_with_highlight(file_path, os.path.join(base_dir, 'target_result.xlsx'))) @@ -348,7 +393,11 @@ class TestUtilsMethods(unittest.TestCase): result_df = pd.DataFrame(data, columns=columns) highlight_dict = {'yellow_rows': [0]} file_path = os.path.join(base_dir, 'result.xlsx') - highlight_rows_xlsx(result_df, highlight_dict, file_path) + + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) + generate_result_xlsx(base_dir) self.assertTrue(compare_excel_files_with_highlight(file_path, os.path.join(base_dir, 'target_result_yellow.xlsx'))) @@ -366,7 +415,9 @@ class TestUtilsMethods(unittest.TestCase): temp_output_file = 'temp_output.txt' sys.stdout = open(temp_output_file, 'w') - highlight_rows_xlsx(result_df, highlight_dict, file_path) + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) with open(temp_output_file, 'r') as f: output = f.read() @@ -391,7 +442,9 @@ class TestUtilsMethods(unittest.TestCase): temp_output_file = 'temp_output.txt' sys.stdout = open(temp_output_file, 'w') - highlight_rows_xlsx(result_df, highlight_dict, file_path) + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.highlight_rows_xlsx(result_df, highlight_dict, file_path) with open(temp_output_file, 'r') as f: output = f.read() @@ -429,7 +482,10 @@ class TestUtilsMethods(unittest.TestCase): 'red_lines': [(0, ['a', 'b'])], 'yellow_lines': [(0, ['c']), (1, ['d'])] } - update_highlight_err_msg(result_df, highlight_dict) + + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.update_highlight_err_msg(result_df, highlight_dict) t_data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', [2, 2], [2, 2], @@ -449,7 +505,9 @@ class TestUtilsMethods(unittest.TestCase): result_df = pd.DataFrame(data, columns=columns) highlight_dict = {} - result = update_highlight_err_msg(result_df, highlight_dict) + mode_config = ModeConfig(dump_mode=Const.MD5) + highlight = HighLight(mode_config) + result = highlight.update_highlight_err_msg(result_df, highlight_dict) self.assertEqual(result, None) @@ -466,5 +524,43 @@ class TestUtilsMethods(unittest.TestCase): 'red_lines': [(0, ['a', 'b'])], 'yellow_lines': [(0, ['c']), (1, ['d'])] } - result = update_highlight_err_msg(result_df, highlight_dict) + mode_config = ModeConfig() + highlight = HighLight(mode_config) + result = highlight.update_highlight_err_msg(result_df, highlight_dict) self.assertEqual(result, None) + + def test_find_error_rows(self): + api_batch = ApiBatch("Functional_batch_norm_0_forward", 0) + api_batch.input_len = 1 + api_batch.output_end_index = 4 + api_batch.params_end_index = 4 + summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] + highlight_dict_test = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} + mode_config = ModeConfig() + highlight = HighLight(mode_config) + highlight.find_error_rows(summary_result, api_batch, highlight_dict_test) + self.assertEqual(highlight_dict_test, + {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []}) + + def test_find_compare_result_error_rows(self): + result = [line_input, line_1, line_2, line_3] + result_df = pd.DataFrame(result) + highlight_dict_test = {"red_rows": set(), "yellow_rows": set(), "red_lines": [], "yellow_lines": []} + mode_config = ModeConfig(dump_mode=Const.ALL) + highlight = HighLight(mode_config) + highlight.find_compare_result_error_rows(result_df, highlight_dict_test) + self.assertEqual(highlight_dict_test, { + "red_rows": {1, 3}, + "yellow_rows": {2}, + "red_lines": [ + (1, ["maximum or minimum is nan, -inf, or inf"]), + (3, ["maximum absolute error exceeds 1e+10"]) + ], + "yellow_lines": [ + (2, ["The output's one thousandth err ratio decreases by more than 0.1 compared to the input/parameter's"]), + (3, [ + "maximum absolute error of both input/parameters and output exceed 1, " + "with the output larger by an order of magnitude", + "The output's cosine decreases by more than 0.1 compared to the input/parameter's"]) + ] + }) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py index 49f084ce07c8e90afb2aa1c3340bb4c3965c8fa7..0180c08e87f6cb78c392223830214fccffb8c149 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py @@ -7,12 +7,12 @@ import unittest import pandas as pd -from msprobe.core.common.const import CompareConst, Const +from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import Comparator, ModeConfig -from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _handle_multi_process, _save_cmp_result, \ - check_accuracy, read_dump_data -from test_acc_compare import generate_dump_json +from msprobe.core.compare.acc_compare import ModeConfig +from msprobe.core.compare.multiprocessing_compute import check_accuracy, CompareRealData, ComparisonResult +from msprobe.pytorch.compare.pt_compare import read_real_data +from test_acc_compare import generate_dump_json, generate_pt, generate_stack_json data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', [2, 2], [2, 2], @@ -28,10 +28,49 @@ columns = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] result_df = pd.DataFrame(data, columns=columns) o_result = pd.DataFrame(o_data, columns=columns) base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_cmp_multiprocessing_compute') +base_dir3 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_acc_compare_data3') +pt_dir = os.path.join(base_dir3, f'dump_data_dir') class TestUtilsMethods(unittest.TestCase): + def test_check_accuracy(self): + max_abs_err = '' + + cos_1 = CompareConst.SHAPE_UNMATCH + result_1 = check_accuracy(cos_1, max_abs_err) + self.assertEqual(result_1, CompareConst.ACCURACY_CHECK_UNMATCH) + + cos_2 = CompareConst.NONE + result_2 = check_accuracy(cos_2, max_abs_err) + self.assertEqual(result_2, CompareConst.NONE) + + cos_3 = 'N/A' + result_3 = check_accuracy(cos_3, max_abs_err) + self.assertEqual(result_3, CompareConst.ACCURACY_CHECK_NO) + + cos_4 = '' + result_4 = check_accuracy(cos_4, max_abs_err) + self.assertEqual(result_4, CompareConst.NONE) + + cos_5 = 0.95 + max_abs_err = 0.002 + result_5 = check_accuracy(cos_5, max_abs_err) + self.assertEqual(result_5, CompareConst.ACCURACY_CHECK_NO) + + cos_6 = 0.85 + max_abs_err = 2 + result_6 = check_accuracy(cos_6, max_abs_err) + self.assertEqual(result_6, CompareConst.ACCURACY_CHECK_NO) + + cos_7 = 0.95 + max_abs_err = 0.001 + result_7 = check_accuracy(cos_7, max_abs_err) + self.assertEqual(result_7, CompareConst.ACCURACY_CHECK_YES) + + +class TestCompareRealData(unittest.TestCase): + def setUp(self): self.result_df = pd.DataFrame(columns=[ CompareConst.COSINE, CompareConst.EUC_DIST, CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR, @@ -39,35 +78,39 @@ class TestUtilsMethods(unittest.TestCase): CompareConst.ACCURACY, CompareConst.ERROR_MESSAGE ]) os.makedirs(base_dir, mode=0o750, exist_ok=True) + os.makedirs(base_dir3, mode=0o750, exist_ok=True) + os.makedirs(pt_dir, mode=0o750, exist_ok=True) self.lock = threading.Lock() def tearDown(self): if os.path.exists(base_dir): shutil.rmtree(base_dir) - - def test_handle_multi_process(self): - stack_mode = False - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - func = Comparator(mode_config).compare_ops - generate_dump_json(base_dir) - input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} - lock = multiprocessing.Manager().RLock() - result = _handle_multi_process(func, input_param, result_df, lock) - self.assertTrue(result.equals(o_result)) + if os.path.exists(pt_dir): + shutil.rmtree(pt_dir) + if os.path.exists(base_dir3): + shutil.rmtree(base_dir3) def test_read_dump_data(self): - result = read_dump_data(result_df) + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + + # normal + result = compare_real_data.read_dump_data(result_df) self.assertEqual(result, {'Functional.linear.0.forward.input.0': ['-1', '-1']}) + # index error with self.assertRaises(CompareException) as context: - result = read_dump_data(pd.DataFrame()) + result = compare_real_data.read_dump_data(pd.DataFrame()) self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) def test_save_cmp_result_success(self): + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + comparison_result = ComparisonResult( cos_result=[0.99, 0.98], max_err_result=[0.01, 0.02], @@ -78,13 +121,18 @@ class TestUtilsMethods(unittest.TestCase): err_msgs=['', 'Error in comparison'] ) offset = 0 - updated_df = _save_cmp_result(offset, comparison_result, self.result_df, self.lock) + updated_df = compare_real_data._save_cmp_result(offset, comparison_result, self.result_df, self.lock) self.assertEqual(updated_df.loc[0, CompareConst.COSINE], 0.99) self.assertEqual(updated_df.loc[1, CompareConst.COSINE], 0.98) self.assertEqual(updated_df.loc[1, CompareConst.ERROR_MESSAGE], 'Error in comparison') def test_save_cmp_result_index_error(self): + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + comparison_result = ComparisonResult( cos_result=[0.99], max_err_result=[], @@ -95,39 +143,108 @@ class TestUtilsMethods(unittest.TestCase): err_msgs=[''] ) with self.assertRaises(CompareException) as context: - _save_cmp_result(0, comparison_result, self.result_df, self.lock) + compare_real_data._save_cmp_result(0, comparison_result, self.result_df, self.lock) self.assertEqual(context.exception.code, CompareException.INDEX_OUT_OF_BOUNDS_ERROR) - def test_check_accuracy(self): - max_abs_err = '' - - cos_1 = CompareConst.SHAPE_UNMATCH - result_1 = check_accuracy(cos_1, max_abs_err) - self.assertEqual(result_1, CompareConst.ACCURACY_CHECK_UNMATCH) - - cos_2 = CompareConst.NONE - result_2 = check_accuracy(cos_2, max_abs_err) - self.assertEqual(result_2, CompareConst.NONE) + def test_compare_by_op_bench_normal(self): + npu_op_name = 'Functional.linear.0.forward.input.0' + bench_op_name = 'Functional.linear.0.forward.input.0' + + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + + pt_name = '-1' + op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} + input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} + result = compare_real_data.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', + 'unsupported', 'No bench data matched.']) + + pt_name = 'Functional.linear.0.forward.input.0.pt' + op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} + input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} + result = compare_real_data.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', + 'unsupported', 'Dump file: Functional.linear.0.forward.input.0.pt not found.']) + + generate_pt(base_dir) + result = compare_real_data.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + self.assertEqual(result, [1.0, 0.0, 0.0, 0.0, 1.0, 1.0, '']) + + def test_compare_by_op_bench_na(self): + npu_op_name = 'Functional.linear.0.forward.input.0' + bench_op_name = 'N/A' + op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [-1, -1]} + input_param = {} + + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + + result = compare_real_data.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) + self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', + 'unsupported', 'No bench data matched.']) + + def test_compare_ops(self): + generate_dump_json(base_dir3) + generate_stack_json(base_dir3) + generate_pt(pt_dir) + dump_path = os.path.join(base_dir3, 'dump.json') + stack_path = os.path.join(base_dir3, 'stack.json') + input_param = {'npu_json_path': dump_path, 'bench_json_path': dump_path, 'stack_json_path': stack_path, + 'is_print_compare_log': True, 'npu_dump_data_dir': pt_dir, 'bench_dump_data_dir': pt_dir} + dump_path_dict = {'Functional.linear.0.forward.input.0': ['Functional.linear.0.forward.input.0.pt', + 'Functional.linear.0.forward.input.0.pt']} + result_df = pd.DataFrame({ + 'NPU Name': ['Functional.linear.0.forward.input.0'], + 'Bench Name': ['Functional.linear.0.forward.input.0'] + }) + + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) + + updated_df = compare_real_data.compare_ops(idx=0, dump_path_dict=dump_path_dict, result_df=result_df, + lock=self.lock, input_param=input_param) + + self.assertEqual(updated_df.loc[0, CompareConst.COSINE], 1.0) + self.assertEqual(updated_df.loc[0, CompareConst.MAX_ABS_ERR], 0) + + def test_do_multi_process(self): + data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', + 'torch.float32', 'torch.float32', [2, 2], [2, 2], + '', '', '', '', '', '', 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', ['-1', '-1']]] + o_data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', + 'torch.float32', 'torch.float32', [2, 2], [2, 2], + 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', + 1, 1, 1, 1, 1, 1, 1, 1, 'None', 'No bench data matched.', ['-1', '-1']]] + columns = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] + result_df = pd.DataFrame(data, columns=columns) + o_result = pd.DataFrame(o_data, columns=columns) + generate_dump_json(base_dir) + input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} - cos_3 = 'N/A' - result_3 = check_accuracy(cos_3, max_abs_err) - self.assertEqual(result_3, CompareConst.ACCURACY_CHECK_NO) + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) - cos_4 = '' - result_4 = check_accuracy(cos_4, max_abs_err) - self.assertEqual(result_4, CompareConst.NONE) - - cos_5 = 0.95 - max_abs_err = 0.002 - result_5 = check_accuracy(cos_5, max_abs_err) - self.assertEqual(result_5, CompareConst.ACCURACY_CHECK_NO) + result = compare_real_data.do_multi_process(input_param, result_df) + self.assertTrue(result.equals(o_result)) - cos_6 = 0.85 - max_abs_err = 2 - result_6 = check_accuracy(cos_6, max_abs_err) - self.assertEqual(result_6, CompareConst.ACCURACY_CHECK_NO) + def test_handle_multi_process(self): + file_reader = read_real_data + mode_config = ModeConfig(dump_mode=Const.ALL) + cross_frame = False + compare_real_data = CompareRealData(file_reader, mode_config, cross_frame) - cos_7 = 0.95 - max_abs_err = 0.001 - result_7 = check_accuracy(cos_7, max_abs_err) - self.assertEqual(result_7, CompareConst.ACCURACY_CHECK_YES) + func = compare_real_data.compare_ops + generate_dump_json(base_dir) + input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} + lock = multiprocessing.Manager().RLock() + result = compare_real_data._handle_multi_process(func, input_param, result_df, lock) + self.assertTrue(result.equals(o_result)) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_postprocess_pass.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_postprocess_pass.py index 9cb33eb277848fa96bdf5b7456867d8579359723..f3623da772d1cbde684aa53119639faa93e4f068 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_postprocess_pass.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_postprocess_pass.py @@ -14,9 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. """ +from dataclasses import dataclass from unittest import TestCase -from msprobe.core.compare.layer_mapping.postprocess_pass import extract_next_item_last_number -from msprobe.core.compare.layer_mapping.postprocess_pass import replace_next_item_index +from msprobe.core.compare.layer_mapping.postprocess_pass import extract_next_item_last_number, \ + replace_next_item_index, renumber_index_pass + + +@dataclass +class DataItem: + """Class for keeping track of an item in inventory""" + type_name: str + full_scope: str + layer_scope: str class TestPostProcessPass(TestCase): @@ -46,3 +55,12 @@ class TestPostProcessPass(TestCase): replace_result = replace_next_item_index(input_data, prefix, inf_value) self.assertEqual(replace_result, input_data) + def test_renumber_index_pass(self): + a = DataItem("ParallelTransformer", "fake_data.layers.10", "fake_data.layers") + b = DataItem("ParallelTransformer", "fake_data.layers.12", "fake_data.layers") + c = DataItem("FakeLayer", "fake_data.layers.10.a.b.c", "fake_data.layers.a.b") + data_items = [a, b, c] + renumber_index_pass(data_items, "ParallelTransformer") + self.assertEqual(a.full_scope, "fake_data.layers.0") + self.assertEqual(b.full_scope, "fake_data.layers.2") + self.assertEqual(c.full_scope, "fake_data.layers.0.a.b.c") diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/bench.sh b/debug/accuracy_tools/msprobe/test/core_ut/config_check/bench.sh new file mode 100644 index 0000000000000000000000000000000000000000..217676ef0f451b6b8f2d2cecb14545d9a7f8dd8b --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/bench.sh @@ -0,0 +1,25 @@ +MASTER_PORT=6000 +NNODES=1 +NODE_RANK=0 +CKPT_SAVE_DIR="your model save ckpt path" +DATA_PATH="your data path" +TOKENIZER_MODEL="your tokenizer path" +CKPT_LOAD_DIR="your model ckpt path" +TP=1 + +DISTRIBUTED_ARGS=" + --master_port $MASTER_PORT +" + +GPT_ARGS=" + --tensor-model-parallel-size ${TP} \ + --sequence-parallel \ + --tokenizer-model ${TOKENIZER_MODEL} \ +" + +torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ + $GPT_ARGS \ + --distributed-backend nccl \ + --load $CKPT_LOAD_DIR \ + --save $CKPT_SAVE_DIR \ + | tee logs/train_llama2_7b.log \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/cmp.sh b/debug/accuracy_tools/msprobe/test/core_ut/config_check/cmp.sh new file mode 100644 index 0000000000000000000000000000000000000000..8df9e6507975c7edbcfee105d838563171c720e4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/cmp.sh @@ -0,0 +1,25 @@ +MASTER_PORT=6001 +NNODES=1 +NODE_RANK=0 +CKPT_SAVE_DIR="./aaa" +DATA_PATH="./aaa" +TOKENIZER_MODEL="./aaa" +CKPT_LOAD_DIR="./aaa" +TP=2 + +DISTRIBUTED_ARGS=" + --master_port $MASTER_PORT +" + +GPT_ARGS=" + --tensor-model-parallel-size ${TP} \ + --sequence-parallel \ + --tokenizer-model ${TOKENIZER_MODEL} \ +" + +torchrun $DISTRIBUTED_ARGS pretrain_gpt.py \ + $GPT_ARGS \ + --distributed-backend nccl \ + --load $CKPT_LOAD_DIR \ + --save $CKPT_SAVE_DIR \ + | tee logs/train_llama2_7b.log \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_ckpt_compare.py b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_ckpt_compare.py new file mode 100644 index 0000000000000000000000000000000000000000..f51aa76f3aed9c8ab2553808563303f5309eb449 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_ckpt_compare.py @@ -0,0 +1,90 @@ +import unittest +from unittest.mock import patch, mock_open +import numpy as np +from msprobe.core.config_check.ckpt_compare import metrics +from msprobe.core.config_check.ckpt_compare import megatron_loader + + + +class TestMetrics(unittest.TestCase): + + def test_in_different_shape(self): + a = np.zeros((2, 3)) + b = np.zeros((2, 3)) + c = np.zeros((3, 2)) + self.assertFalse(metrics.in_different_shape(a, b)) + self.assertTrue(metrics.in_different_shape(a, c)) + + def test_l2_distance(self): + a = np.array([1.0, 2.0, 3.0]) + b = np.array([1.0, 2.0, 3.0]) + c = np.array([4.0, 5.0, 6.0]) + self.assertAlmostEqual(metrics.l2_distance(a, b), 0.0) + self.assertAlmostEqual(metrics.l2_distance(a, c), np.linalg.norm(a - c)) + self.assertIsNone(metrics.l2_distance(None, b)) + self.assertIsNone(metrics.l2_distance(a, None)) + self.assertIsNone(metrics.l2_distance(a, np.zeros((2, 2)))) + + def test_cos_sim(self): + a = np.array([1.0, 0.0, 0.0], dtype=np.float32) + b = np.array([1.0, 0.0, 0.0], dtype=np.float32) + c = np.array([0.0, 1.0, 0.0], dtype=np.float32) + + self.assertAlmostEqual(metrics.cos_sim(a, b), 1.0, places=6) + self.assertAlmostEqual(metrics.cos_sim(a, c), 0.0, places=6) + self.assertIsNone(metrics.cos_sim(a, np.zeros((2, 2), dtype=np.float32))) + + def test_numel(self): + a = np.zeros((2, 3)) + b = np.zeros((2, 3)) + c = np.zeros((3, 2)) + self.assertEqual(metrics.numel(a, b), 6) + self.assertEqual(metrics.numel(a, c), 6) + d = np.zeros((2, 2)) + self.assertEqual(metrics.numel(a, d), (6, 4)) + + def test_shape(self): + a = np.zeros((2, 3)) + b = np.zeros((2, 3)) + c = np.zeros((3, 2)) + self.assertEqual(metrics.shape(a, b), [2, 3]) + self.assertEqual(metrics.shape(a, c), [[2, 3], [3, 2]]) + + +class TestMegatronLoader(unittest.TestCase): + + def test__parse_real_layer_idx(self): + name = 'layers.2.attn/1' # vpp_stage = 1 + result = megatron_loader._parse_real_layer_idx(name, num_layers_per_stage=4, pp_size=2, pp_rank=1) + self.assertEqual(result, 'layers.14.attn') + + def test__parse_real_expert_idx(self): + name = 'layers.0.experts.3.weight' + result = megatron_loader._parse_real_expert_idx(name, num_experts_per_rank=4, exp_rank=2) + self.assertIn('experts.11', result) # 3 + 2*4 = 11 + + # No expert pattern + name2 = 'layers.0.weight' + self.assertEqual(megatron_loader._parse_real_expert_idx(name2, 4, 2), name2) + + def test__consolidate_tp_weights(self): + arr1 = np.ones((2,2)) + arr2 = np.zeros((2,2)) + weights = { + 'linear_fc1.weight': [arr1, arr2], + 'linear_fc2.weight': [arr1, arr2], + 'linear_fc2.bias': [arr1, arr1] + } + result = megatron_loader._consolidate_tp_weights(weights) + self.assertTrue(np.allclose(result['linear_fc1.weight'], np.concatenate([arr1, arr2], axis=0))) + self.assertTrue(np.allclose(result['linear_fc2.weight'], np.concatenate([arr1, arr2], axis=1))) + self.assertTrue(np.allclose(result['linear_fc2.bias'], arr1)) + + def test__parse_num_layers_per_stage(self): + keys = {'layers.0.weight': None, 'layers.1.weight': None, 'layers.2.weight': None} + self.assertEqual(megatron_loader._parse_num_layers_per_stage(keys), 3) + + +if __name__ == '__main__': + unittest.main() + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_config_check.py b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_config_check.py new file mode 100644 index 0000000000000000000000000000000000000000..9234cf0e0076a9bb268d96120d3de813c53a7c29 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_config_check.py @@ -0,0 +1,173 @@ +import os +import random +import shutil +import unittest +import torch +import json +import numpy as np +import torch.nn as nn +import mindspore as ms +import mindspore.nn as ms_nn +from mindspore import Tensor +from msprobe.core.config_check.config_checker import ConfigChecker +from msprobe.core.config_check.checkers.pip_checker import PipPackageChecker +from msprobe.core.config_check.checkers.random_checker import RandomChecker +from msprobe.core.config_check.checkers.dataset_checker import DatasetChecker +from msprobe.core.config_check.checkers.weights_checker import WeightsChecker +from msprobe.core.common.file_utils import read_xlsx +from msprobe.core.common.framework_adapter import FmkAdp + + +testdir = os.path.dirname(__file__) +config_checking_dir = os.path.dirname(testdir) +temp_dir = os.path.join(config_checking_dir, "temp") +os.makedirs(temp_dir, exist_ok=True) +ms.set_context(device_target="CPU") + + +def seed_all(seed=1234, mode=False): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.use_deterministic_algorithms(mode) + ms.set_seed(seed) + + +class MockPyTorchModule(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(10, 5) + self.relu = nn.ReLU() + + def forward(self, x, y): + x1 = self.linear(x) + x2 = self.relu(x1) + return x2 + + +class MockMindSporeModule(ms_nn.Cell): + def __init__(self): + super().__init__() + self.linear = ms_nn.Dense(10, 5) + self.relu = ms_nn.ReLU() + + def construct(self, x): + x1 = self.linear(x) + x2 = self.relu(x1) + return x2 + + +def get_test_dataset(): + inputs = [torch.rand(10, 10) for _ in range(10)] + labels = [torch.randint(0, 5, (10,)) for _ in range(10)] + ms_inputs = [Tensor(input.numpy()) for input in inputs] + ms_labels = [Tensor(label.numpy()) for label in labels] + return zip(inputs, labels), zip(ms_inputs, ms_labels) + + +def get_test_model(use_pytorch=True): + if use_pytorch: + test_module = MockPyTorchModule() + nn.init.constant_(test_module.linear.weight, 1.0) + nn.init.constant_(test_module.linear.bias, 1.0) + return test_module + else: + test_module = MockMindSporeModule() + for param in test_module.get_parameters(): + param.set_data(ms.Tensor(np.ones(param.data.shape), dtype=param.data.dtype)) + return test_module + + +@unittest.mock.patch("msprobe.core.config_check.checkers.pip_checker.collect_pip_data") +@unittest.mock.patch("msprobe.core.config_check.checkers.env_args_checker.collect_env_data") +def train_test(seed, output_zip_path, shell_path, mock_env, mock_pip): + if seed == 1234: + mock_pip.return_value = "transformers=0.0.1" + mock_env.return_value = {"NCCL_DETERMINISTIC": True} + else: + mock_pip.return_value = "transformers=0.0.2" + mock_env.return_value = {"HCCL_DETERMINISTIC": False, "ASCEND_LAUNCH_BLOCKING": 1} + seed_all(seed) + + use_pytorch = seed == 1234 + test_dataset, ms_test_dataset = get_test_dataset() + test_module = get_test_model(use_pytorch) + + if use_pytorch: + loss_fun = nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(test_module.parameters(), lr=1e-2) + ConfigChecker(test_module, shell_path, output_zip_path) + + for input_data, label in test_dataset: + output = test_module(input_data, y=input_data) + loss = loss_fun(output, label) + optimizer.zero_grad() + loss.backward() + optimizer.step() + + else: + loss_fun = ms_nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') + optimizer = ms_nn.SGD(test_module.trainable_params(), learning_rate=1e-2) + train_network = ms_nn.TrainOneStepCell(ms_nn.WithLossCell(test_module, loss_fun), optimizer) + ConfigChecker(test_module, shell_path, output_zip_path, fmk="mindspore") + + for input_data, label in ms_test_dataset: + loss = train_network(input_data, label) + + + +class TestConfigChecker(unittest.TestCase): + def tearDown(self): + FmkAdp.set_fmk("pytorch") + shutil.rmtree(temp_dir) + + + def test_all(self): + train_test(1234, os.path.join(temp_dir, "config_check_pack1.zip"), [os.path.join(testdir, "cmp.sh")]) + + ConfigChecker.pre_forward_fun_list = [] + ConfigChecker.step = 0 + RandomChecker.write_once = False + ConfigChecker.apply_patches("pytorch") + ConfigChecker.apply_patches("mindspore") + + train_test(1233, os.path.join(temp_dir, "config_check_pack2.zip"), [os.path.join(testdir, "bench.sh")]) + + ConfigChecker.compare(os.path.join(temp_dir, "config_check_pack1.zip"), + os.path.join(temp_dir, "config_check_pack2.zip"), + os.path.join(temp_dir, "compare_output")) + + compare_output_dir = os.path.join(temp_dir, "compare_output") + + total_check_result = read_xlsx(os.path.join(compare_output_dir, ConfigChecker.result_filename)) + self.assertEqual(total_check_result.columns.tolist(), ConfigChecker.result_header) + target_total_check_result = [ + ['env', False], + ['pip', False], + ['dataset', False], + ['weights', False], + ['hyperparameters', False], + ['random', False] + ] + self.assertEqual(total_check_result.values.tolist(), target_total_check_result) + + pip_data_check_result = read_xlsx(os.path.join(compare_output_dir, ConfigChecker.result_filename), + sheet_name=PipPackageChecker.target_name_in_zip) + self.assertEqual(pip_data_check_result.columns.tolist(), PipPackageChecker.result_header) + self.assertEqual(pip_data_check_result.iloc[0].tolist(), ['transformers', '0.0.1', '0.0.2', 'error']) + + random_check_result = read_xlsx(os.path.join(compare_output_dir, ConfigChecker.result_filename), + sheet_name=RandomChecker.target_name_in_zip) + self.assertEqual(random_check_result.columns.tolist(), RandomChecker.result_header) + self.assertEqual(len(random_check_result), 5) + + dataset_check_result = read_xlsx(os.path.join(compare_output_dir, ConfigChecker.result_filename), + sheet_name=DatasetChecker.target_name_in_zip) + self.assertEqual(dataset_check_result.columns.tolist(), DatasetChecker.result_header) + self.assertEqual(len(dataset_check_result), 20) + + weight_check_result = read_xlsx(os.path.join(compare_output_dir, ConfigChecker.result_filename), + sheet_name=WeightsChecker.target_name_in_zip) + self.assertEqual(weight_check_result.columns.tolist(), WeightsChecker.result_header) + self.assertEqual(len(weight_check_result), 20) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_dataset_checker.py b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_dataset_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..27db8e04d0890d579fae4ec02a7260102f08979b --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_dataset_checker.py @@ -0,0 +1,76 @@ +import unittest +import torch +import pandas as pd +from unittest.mock import patch, MagicMock + +from msprobe.core.config_check.checkers.dataset_checker import compare_dataset, \ + compare_dataset_dicts, parse_args_and_kargs, process_obj + + +class TestTensorProcessing(unittest.TestCase): + + def test_process_obj_tensor(self): + tensor = torch.tensor([1.0, 2.0, 3.0]) + result = process_obj(tensor) + self.assertEqual(isinstance(result, dict), True) + self.assertEqual(set(result.keys()), {'max', 'min', 'mean', 'norm'}) + + def test_process_obj_list(self): + obj = [torch.tensor([1.0]), torch.tensor([2.0])] + result = process_obj(obj) + self.assertEqual(isinstance(result, dict), True) + self.assertEqual(set(result.keys()), {0, 1}) + + def test_process_obj_dict(self): + obj = {'a': torch.tensor([1.0]), 'b': torch.tensor([2.0])} + result = process_obj(obj) + self.assertEqual(isinstance(result, dict), True) + self.assertEqual(set(result.keys()), {'a', 'b'}) + + def test_process_obj_other(self): + obj = "test" + result = process_obj(obj) + self.assertEqual(result, "") + + def test_parse_args_and_kargs(self): + args = (torch.tensor([1.0]),) + kwargs = {'a': torch.tensor([2.0])} + result = parse_args_and_kargs(args, kwargs) + self.assertEqual(isinstance(result, dict), True) + self.assertEqual(set(result.keys()), {'args', 'kwargs'}) + + def test_compare_dataset_dicts_equal(self): + dict1 = {'a': {'max': 1.0, 'min': 0.0, 'mean': 0.5, 'norm': 0.7}} + dict2 = {'a': {'max': 1.0, 'min': 0.0, 'mean': 0.5, 'norm': 0.7}} + results = compare_dataset_dicts(dict1, dict2) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['equal'], True) + + def test_compare_dataset_dicts_not_equal(self): + dict1 = {'a': {'max': 1.0, 'min': 0.0, 'mean': 0.5, 'norm': 0.7}} + dict2 = {'a': {'max': 2.0, 'min': 0.0, 'mean': 0.5, 'norm': 0.7}} + results = compare_dataset_dicts(dict1, dict2) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['equal'], False) + + def test_compare_dataset_dicts_nested(self): + dict1 = {'a': {'b': {'max': 1.0, 'min': 0.0, 'mean': 0.5, 'norm': 0.7}}} + dict2 = {'a': {'b': {'max': 1.0, 'min': 0.0, 'mean': 0.5, 'norm': 0.7}}} + results = compare_dataset_dicts(dict1, dict2) + self.assertEqual(len(results), 1) + self.assertEqual(results[0]['tag'], 'a.b') + + @patch('os.listdir', side_effect=[["step1"], ["rank1"]]) + @patch('os.path.isdir', return_value=True) + @patch('os.path.isfile', return_value=True) + @patch('msprobe.core.config_check.checkers.dataset_checker.load_json') + def test_compare_dataset(self, mock_load_json, mock_isfile, mock_isdir, mock_listdir): + mock_load_json.return_value = {'a': {'max': 1.0, 'min': 0.0, 'mean': 0.5, 'norm': 0.7}} + bench_dir = 'bench' + cmp_dir = 'cmp' + result = compare_dataset(bench_dir, cmp_dir) + self.assertEqual(isinstance(result, pd.DataFrame), True) + + + + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_random_checker.py b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_random_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..9a6bdb89f83d0c154c86d6522ffc76bb042cff4d --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_random_checker.py @@ -0,0 +1,77 @@ +import unittest +import pandas as pd +from unittest.mock import patch, MagicMock + +from msprobe.core.config_check.checkers.random_checker import compare_json_files, compare_random, get_file_and_line + + +class TestCompareRandom(unittest.TestCase): + + @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) + @patch('os.path.join', return_value='test_path') + @patch("msprobe.core.config_check.checkers.random_checker.load_json") + def test_compare_random_with_files(self, mock_load_json, mock_path, mock_listdir): + mock_load_json.return_value = {"op1": {"position1": 1}} + bench_dir = 'test_bench' + cmp_dir = 'test_cmp' + result = compare_random(bench_dir, cmp_dir) + self.assertEqual(isinstance(result, pd.DataFrame), True) + + @patch('os.listdir', return_value=[]) + @patch('os.path.join', return_value='test_path') + def test_compare_random_no_files(self, mock_path, mock_listdir): + bench_dir = 'test_bench' + cmp_dir = 'test_cmp' + result = compare_random(bench_dir, cmp_dir) + self.assertEqual(isinstance(result, pd.DataFrame), True) + self.assertEqual(len(result), 0) + + def test_get_file_and_line_with_valid_input(self): + position = '/path/to/file.py:10' + result = get_file_and_line(position) + self.assertEqual(isinstance(result, str), True) + self.assertEqual(result, 'file.py:10') + + def test_get_file_and_line_with_invalid_input(self): + position = 'invalid_position' + result = get_file_and_line(position) + self.assertEqual(isinstance(result, str), True) + self.assertEqual(result, 'invalid_position') + + @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) + @patch('os.path.join', return_value='test_path') + def test_compare_json_files_same_data(self, mock_path, mock_listdir): + bench_data = {"op1": {"position1:10": 1}} + cmp_data = {"op1": {"position1:10": 1}} + result = compare_json_files(bench_data, cmp_data) + self.assertEqual(isinstance(result, list), True) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][2], True) + + @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) + @patch('os.path.join', return_value='test_path') + def test_compare_json_files_different_data(self, mock_path, mock_listdir): + bench_data = {"op1": {"position1:10": 1}} + cmp_data = {"op1": {"position1:10": 2}} + result = compare_json_files(bench_data, cmp_data) + self.assertEqual(isinstance(result, list), True) + self.assertEqual(len(result), 1) + self.assertEqual(result[0][2], False) + + @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) + @patch('os.path.join', return_value='test_path') + def test_compare_json_files_missing_op_in_bench(self, mock_path, mock_listdir): + bench_data = {} + cmp_data = {"op1": {"position1:10": 1}} + result = compare_json_files(bench_data, cmp_data) + self.assertEqual(isinstance(result, list), True) + self.assertEqual(len(result), 1) + + @patch('os.listdir', return_value=['rank1.json', 'rank2.json']) + @patch('os.path.join', return_value='test_path') + def test_compare_json_files_missing_op_in_cmp(self, mock_path, mock_listdir): + bench_data = {"op1": {"position1:10": 1}} + cmp_data = {} + result = compare_json_files(bench_data, cmp_data) + self.assertEqual(isinstance(result, list), True) + self.assertEqual(len(result), 1) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_weight_checker.py b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_weight_checker.py new file mode 100644 index 0000000000000000000000000000000000000000..4920c034455920075c617f7db90a26fbe826b10c --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/config_check/test_weight_checker.py @@ -0,0 +1,79 @@ +import unittest +from unittest.mock import patch +import pandas as pd +import os +import torch + +from msprobe.core.config_check.checkers.weights_checker import collect_weights_data, compare_weight, compare_weight_file + + +class TestWeightComparison(unittest.TestCase): + @patch('msprobe.core.config_check.utils.utils.get_tensor_features') + @patch('torch.nn.Module.named_parameters') + def test_collect_weights_data(self, mock_named_parameters, mock_get_tensor_features): + mock_model = unittest.mock.create_autospec(torch.nn.Module) + mock_named_parameters.return_value = [('param1', object())] + mock_get_tensor_features.return_value = {'max': 1, 'min': 0, 'mean': 0.5, 'norm': 1} + result = collect_weights_data(mock_model) + self.assertEqual(isinstance(result, dict), True) + + @patch('msprobe.core.config_check.checkers.weights_checker.load_json') + def test_compare_weight_file(self, mock_load_json): + mock_load_json.side_effect = [ + {'weight1': {'max': 1, 'min': 0, 'mean': 0.5, 'norm': 1}}, + {'weight1': {'max': 1, 'min': 0, 'mean': 0.5, 'norm': 1}} + ] + result = compare_weight_file('bench.json', 'cmp.json') + self.assertEqual(isinstance(result, list), True) + + @patch('msprobe.core.config_check.checkers.weights_checker.os_walk_for_files') + @patch('msprobe.core.config_check.checkers.weights_checker.load_json') + @patch('os.path.exists') + def test_compare_weight(self, mock_exists, mock_load_json, mock_os_walk_for_files): + mock_os_walk_for_files.return_value = [ + {"root": "bench/step1/rank0", "file": "weights.json"} + ] + mock_load_json.return_value = {'weight1': {'max': 1, 'min': 0, 'mean': 0.5, 'norm': 1}} + mock_exists.return_value = True + result = compare_weight('bench', 'cmp') + self.assertEqual(isinstance(result, pd.DataFrame), True) + + @patch('msprobe.core.config_check.checkers.weights_checker.load_json') + def test_compare_weight_file_different_weights(self, mock_load_json): + mock_load_json.side_effect = [ + {'weight1': {'max': 1, 'min': 0, 'mean': 0.5, 'norm': 1}}, + {'weight1': {'max': 2, 'min': 1, 'mean': 1.5, 'norm': 2}} + ] + result = compare_weight_file('bench.json', 'cmp.json') + self.assertEqual(isinstance(result, list), True) + for res in result: + if res["weight_name"] == "weight1": + self.assertEqual(res["equal"], False) + + @patch('msprobe.core.config_check.checkers.weights_checker.os_walk_for_files') + @patch('msprobe.core.config_check.checkers.weights_checker.load_json') + @patch('os.path.exists') + def test_compare_weight_cmp_file_missing(self, mock_exists, mock_load_json, mock_os_walk_for_files): + mock_os_walk_for_files.return_value = [ + {"root": "bench/step1/rank0", "file": "weights.json"} + ] + mock_load_json.return_value = {'weight1': {'max': 1, 'min': 0, 'mean': 0.5, 'norm': 1}} + mock_exists.return_value = False + result = compare_weight('bench', 'cmp') + self.assertEqual(isinstance(result, pd.DataFrame), True) + self.assertEqual(len(result[result["equal"] == "only bench have"]), 1) + + @patch('msprobe.core.config_check.checkers.weights_checker.os_walk_for_files') + @patch('msprobe.core.config_check.checkers.weights_checker.load_json') + @patch('os.path.exists') + def test_compare_weight_multiple_files(self, mock_exists, mock_load_json, mock_os_walk_for_files): + mock_os_walk_for_files.return_value = [ + {"root": "bench/step1/rank0", "file": "weights1.json"}, + {"root": "bench/step1/rank0", "file": "weights2.json"} + ] + mock_load_json.return_value = {'weight1': {'max': 1, 'min': 0, 'mean': 0.5, 'norm': 1}} + mock_exists.return_value = True + result = compare_weight('bench', 'cmp') + self.assertEqual(isinstance(result, pd.DataFrame), True) + self.assertEqual(len(result), 2) + diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py index 8ff89437646ee203aaa4a3fac5bbfea1538e9409..f9b6bd4d8a2266e0f449239a7df87d5caf9d1b10 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py @@ -70,31 +70,23 @@ class TestBaseDataProcessor(unittest.TestCase): @patch('inspect.stack') def test_analyze_api_call_stack(self, mock_stack): mock_stack.return_value = [ - (None, 'file0.py', 0, 'function0', ['code line 0'], None), - (None, 'file1.py', 10, 'function1', ['code line 1'], None), - (None, 'file2.py', 20, 'function2', ['code line 2'], None), (None, 'file3.py', 30, 'function3', ['code line 3'], None), - (None, 'file4.py', 40, 'function4', ['code line 4'], None), - (None, 'file5.py', 50, 'function5', ['code line 5'], None), - (None, 'file6.py', 60, 'function6', ['code line 6'], None), - (None, 'file7.py', 70, 'function7', ['code line 7'], None), + (None, 'file1.py', 40, 'function1', ['code line 1'], None), + (None, 'file2.py', 50, 'function2', ['code line 2'], None), + (None, 'file3.py', 60, 'function3', ['code line 3'], None), + (None, 'file1.py', 70, 'function1', ['code line 1'], None), + (None, 'file1.py', 80, 'function1', ['code line 1'], None), + (None, 'file2.py', 90, 'function2', ['code line 2'], None), + (None, 'file3.py', 100, 'function3', ['code line 3'], None) ] result = BaseDataProcessor.analyze_api_call_stack('test_stack') - expected_output = { - 'test_stack': [ - 'File file5.py, line 50, in function5, \n code line 5', - 'File file6.py, line 60, in function6, \n code line 6', - 'File file7.py, line 70, in function7, \n code line 7', - ] - } - self.assertEqual(result, expected_output) + expected_output = ( + 'File file1.py, line 80, in function1, \n code line 1', + 'File file2.py, line 90, in function2, \n code line 2', + 'File file3.py, line 100, in function3, \n code line 3', + ) - def test_convert_numpy_to_builtin(self): - self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.int32(5)), (5, 'int32')) - self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.float64(3.14)), (3.14, 'float64')) - self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.bool_(True)), (True, 'bool_')) - self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.str_('test')), ('test', 'str_')) - self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(5), (5, '')) + self.assertEqual(result, expected_output) def test_analyze_builtin(self): result = self.processor._analyze_builtin(slice(1, 10, 2)) @@ -113,12 +105,37 @@ class TestBaseDataProcessor(unittest.TestCase): expected = {'type': 'int', 'value': 1} self.assertEqual(result, expected) + def test_analyze_numpy(self): + result = BaseDataProcessor._analyze_numpy(np.int32(5)) + expected = {"type": 'int32', "value": 5} + self.assertEqual(result, expected) + + result = BaseDataProcessor._analyze_numpy(np.float32(3.14)) + expected = {"type": 'float32', "value": 3.140000104904175} + self.assertEqual(result, expected) + + result = BaseDataProcessor._analyze_numpy(np.bool_(True)) + expected = {"type": 'bool_', "value": True} + self.assertEqual(result, expected) + + result = BaseDataProcessor._analyze_numpy(np.str_("abc")) + expected = {"type": 'str_', "value": "abc"} + self.assertEqual(result, expected) + + result = BaseDataProcessor._analyze_numpy(np.byte(1)) + expected = {"type": 'int8', "value": 1} + self.assertEqual(result, expected) + + result = BaseDataProcessor._analyze_numpy(np.complex128(1 + 2j)) + expected = {"type": 'complex128', "value": (1 + 2j)} + self.assertEqual(result, expected) + def test_get_special_types(self): self.assertIn(int, BaseDataProcessor.get_special_types()) - def test_analyze_numpy(self): + def test_analyze_ndarray(self): ndarray = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.int32) - result = BaseDataProcessor._analyze_numpy(ndarray, 'numpy.ndarray') + result = BaseDataProcessor._analyze_ndarray(ndarray, 'numpy.ndarray') expected_result = { 'type': 'numpy.ndarray', 'dtype': 'int32', @@ -126,7 +143,20 @@ class TestBaseDataProcessor(unittest.TestCase): 'Max': 6, 'Min': 1, 'Mean': 3.5, - 'Norm':9.539392014169456 + 'Norm': 9.539392014169456 + } + self.assertEqual(result, expected_result) + + ndarray = np.array([], dtype=np.int32) + result = BaseDataProcessor._analyze_ndarray(ndarray, 'numpy.ndarray') + expected_result = { + 'type': 'numpy.ndarray', + 'dtype': 'int32', + 'shape': (0,), + 'Max': None, + 'Min': None, + 'Mean': None, + 'Norm': None } self.assertEqual(result, expected_result) @@ -134,6 +164,7 @@ class TestBaseDataProcessor(unittest.TestCase): transform = lambda x, _: x * 2 Test = namedtuple("Test", ['a']) myNamedTuple = Test(1) + @dataclass class MyDataClass: last_hidden_state: int = None @@ -145,7 +176,7 @@ class TestBaseDataProcessor(unittest.TestCase): hidden_states=(2, 3), attentions=(4, 5) ) - expected_dataclass_res = {'last_hidden_state': 2, 'hidden_states': [4, 6], 'attentions': [8,10]} + expected_dataclass_res = {'last_hidden_state': 2, 'hidden_states': [4, 6], 'attentions': [8, 10]} self.assertEqual(BaseDataProcessor.recursive_apply_transform(2, transform), 4) self.assertEqual(BaseDataProcessor.recursive_apply_transform(myData, transform), expected_dataclass_res) self.assertEqual(BaseDataProcessor.recursive_apply_transform(myNamedTuple, transform), {'a': 2}) @@ -280,9 +311,9 @@ class TestBaseDataProcessor(unittest.TestCase): self.assertEqual(dst_data_structure, excepted_result) def test_analyze_element_to_all_none(self): - element = {"key1": [12, 3, {"key2": 10, "key3":["12"]}]} + element = {"key1": [12, 3, {"key2": 10, "key3": ["12"]}]} result = self.processor.analyze_element_to_all_none(element) - excepted_result = {"key1": [None, None, {"key2": None, "key3":[None]}]} + excepted_result = {"key1": [None, None, {"key2": None, "key3": [None]}]} self.assertEqual(result, excepted_result) @patch.object(MindsporeDataProcessor, "is_hookable_element", return_value=True) @@ -327,4 +358,4 @@ class TestBaseDataProcessor(unittest.TestCase): nested_data_structure, ["grad_name_1", "layer1", "layer2"], "grad_data_info" ) self.assertIsNone(self.processor.save_name) - self.assertEqual(result, grad) \ No newline at end of file + self.assertEqual(result, grad) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py index b593d34c5d86c7fb3b4a0e8a3ff548c55555e09d..25141a9e774d6a6ca05be9b668de96a7d57cb373 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -19,9 +19,10 @@ from unittest.mock import patch, MagicMock import zlib import mindspore as ms -from mindspore import Tensor +from mindspore import Tensor, ops, mint import numpy as np +from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import BaseDataProcessor from msprobe.core.data_dump.data_processor.mindspore_processor import ( MindsporeDataProcessor, @@ -32,6 +33,13 @@ from msprobe.core.data_dump.data_processor.mindspore_processor import ( from msprobe.mindspore.common.log import logger +def patch_norm(value): + return ops.norm(value) + + +setattr(mint, "norm", patch_norm) + + class TestMindsporeDataProcessor(unittest.TestCase): def setUp(self): self.config = MagicMock() @@ -69,11 +77,16 @@ class TestMindsporeDataProcessor(unittest.TestCase): def test_get_stat_info_float_async(self): self.config.async_dump = True tensor = ms.tensor([1.0, 2.0, 3.0]) - result = self.processor.get_stat_info(tensor).stack_tensor_stat[1] - self.assertEqual(result[0].item(), 3.0) - self.assertEqual(result[1].item(), 1.0) - self.assertEqual(result[2].item(), 2.0) - self.assertEqual(result[3].item(), ms.ops.norm(tensor).item()) + result = self.processor.get_stat_info(tensor) + result_max = result.max + result_min = result.min + result_mean = result.mean + result_norm = result.norm + + self.assertEqual(result_max.item(), 3.0) + self.assertEqual(result_min.item(), 1.0) + self.assertEqual(result_mean.item(), 2.0) + self.assertEqual(result_norm.item(), ms.ops.norm(tensor).item()) def test_get_stat_info_int(self): self.config.async_dump = False @@ -87,9 +100,13 @@ class TestMindsporeDataProcessor(unittest.TestCase): def test_get_stat_info_int_async(self): self.config.async_dump = True tensor = ms.tensor([1, 2, 3]) - result = self.processor.get_stat_info(tensor).stack_tensor_stat[1] - self.assertEqual(result[0].item(), 3.0) - self.assertEqual(result[1].item(), 1.0) + result = self.processor.get_stat_info(tensor) + + result_max = result.max + result_min = result.min + + self.assertEqual(result_max.item(), 3.0) + self.assertEqual(result_min.item(), 1.0) def test_get_stat_info_bool(self): self.config.async_dump = False @@ -103,9 +120,13 @@ class TestMindsporeDataProcessor(unittest.TestCase): def test_get_stat_info_bool_async(self): self.config.async_dump = True tensor = ms.Tensor([True, False, True]) - result = self.processor.get_stat_info(tensor).stack_tensor_stat[1] - self.assertEqual(result[0].item(), True) - self.assertEqual(result[1].item(), False) + result = self.processor.get_stat_info(tensor) + + result_max = result.max + result_min = result.min + + self.assertEqual(result_max.item(), True) + self.assertEqual(result_min.item(), False) @patch.object(MindsporeDataProcessor, 'get_md5_for_tensor') def test__analyze_tensor(self, get_md5_for_tensor): @@ -118,13 +139,12 @@ class TestMindsporeDataProcessor(unittest.TestCase): 'type': 'mindspore.Tensor', 'dtype': 'Int32', 'shape': (3,), - 'Max': 3, - 'Min': 1, - 'Mean': 2, - 'Norm': ms.ops.norm(tensor).item(), 'md5': 'test_md5', } result = self.processor._analyze_tensor(tensor, suffix) + # 删除不必要的字段 + result.pop('tensor_stat_index', None) + self.assertEqual(result, expected_result) @@ -150,12 +170,9 @@ class TestTensorDataProcessor(unittest.TestCase): 'type': 'mindspore.Tensor', 'dtype': str(tensor.dtype), 'shape': tensor.shape, - 'Max': 3.0, - 'Min': 1.0, - 'Mean': 2.0, - 'Norm': ms.ops.norm(tensor).item(), 'data_name': 'test_api.input.suffix.npy' } + result.pop('tensor_stat_index', None) self.assertEqual(expected, result) @@ -164,6 +181,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): class Config: def __init__(self): self.overflow_nums = 1 + self.data_processor = OverflowCheckDataProcessor(Config(), None) def test___init__(self): @@ -174,6 +192,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): def test_analyze_forward(self): def func(_): self.data_processor.has_overflow = True + with patch.object(BaseDataProcessor, "analyze_forward", return_value={"min", 0}): with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data"): api_info = self.data_processor.analyze_forward("name", "module", "module_input_output") @@ -187,6 +206,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): def test_analyze_backward(self): def func(_): self.data_processor.has_overflow = True + with patch.object(BaseDataProcessor, "analyze_backward", return_value={"min", 0}): with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data"): api_info = self.data_processor.analyze_backward("name", "module", "module_input_output") @@ -218,33 +238,62 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.data_processor.overflow_nums = 3 self.assertFalse(self.data_processor.is_terminated) + # from unittest.mock import MagicMock + def test__analyze_maybe_overflow_tensor(self): + # Mock DataWriter 和相关方法 + self.data_processor.data_writer = MagicMock() + + tensor_json = {Const.TENSOR_STAT_INDEX: 1} # 修正：添加正确的 tensor_stat_index + + # 模拟返回值 + self.data_processor.data_writer.get_buffer_values_max.return_value = 10 + self.data_processor.data_writer.get_buffer_values_min.return_value = -10 + self.data_processor.has_overflow = False - tensor_json = {"Max": None, "Min": 0} + # 调用函数并检查没有溢出 self.data_processor._analyze_maybe_overflow_tensor(tensor_json) self.assertFalse(self.data_processor.has_overflow) - tensor_json.update({"Max": -np.inf}) + + self.data_processor.has_overflow = False + # max 值为 -np.inf，应该触发溢出 + self.data_processor.data_writer.get_buffer_values_max.return_value = -np.inf self.data_processor._analyze_maybe_overflow_tensor(tensor_json) self.assertTrue(self.data_processor.has_overflow) + self.data_processor.has_overflow = False - tensor_json.update({"Max": np.inf}) + # max 值为 np.inf，应该触发溢出 + self.data_processor.data_writer.get_buffer_values_max.return_value = np.inf self.data_processor._analyze_maybe_overflow_tensor(tensor_json) self.assertTrue(self.data_processor.has_overflow) + self.data_processor.has_overflow = False - tensor_json.update({"Max": np.nan}) + # max 值为 np.nan，应该触发溢出 + self.data_processor.data_writer.get_buffer_values_max.return_value = np.nan self.data_processor._analyze_maybe_overflow_tensor(tensor_json) self.assertTrue(self.data_processor.has_overflow) - tensor_json.update({"Max": 0}) + + self.data_processor.has_overflow = False + # max 值为 0，不会触发溢出 + self.data_processor.data_writer.get_buffer_values_max.return_value = 0 + self.data_processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertFalse(self.data_processor.has_overflow) + self.data_processor.has_overflow = False - tensor_json.update({"Min": -np.inf}) + # min 值为 -np.inf，应该触发溢出 + self.data_processor.data_writer.get_buffer_values_min.return_value = -np.inf self.data_processor._analyze_maybe_overflow_tensor(tensor_json) self.assertTrue(self.data_processor.has_overflow) + self.data_processor.has_overflow = False - tensor_json.update({"Min": np.inf}) + # min 值为 np.inf，应该触发溢出 + self.data_processor.data_writer.get_buffer_values_min.return_value = np.inf self.data_processor._analyze_maybe_overflow_tensor(tensor_json) self.assertTrue(self.data_processor.has_overflow) + self.data_processor.has_overflow = False - tensor_json.update({"Min": np.nan}) + # min 值为 np.nan，应该触发溢出 + self.data_processor.data_writer.get_buffer_values_min.return_value = np.nan self.data_processor._analyze_maybe_overflow_tensor(tensor_json) self.assertTrue(self.data_processor.has_overflow) @@ -260,7 +309,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): return_value=False): ret = self.data_processor._analyze_tensor("tensor", "suffix") self.assertEqual(self.data_processor.cached_tensors_and_file_paths, {"file_path": "tensor"}) - mock_warning.assert_not_called() + mock_warning.assert_called_with("tensor_stat_index does not exist in tensor_json.") mock_super.assert_called_with("tensor", "suffix") self.assertEqual(ret.get("Max"), None) self.assertEqual(ret.get("data_name"), "dump_data_name") @@ -268,7 +317,8 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): with patch("msprobe.core.data_dump.data_processor.mindspore_processor.path_len_exceeds_limit", return_value=True): self.data_processor._analyze_tensor("tensor", "suffix") - mock_warning.assert_called_with("The file path file_path length exceeds limit.") + mock_warning.assert_called_with("tensor_stat_index does not exist in tensor_json.") + class TestKernelDumpDataProcessor(unittest.TestCase): def setUp(self): @@ -293,7 +343,8 @@ class TestKernelDumpDataProcessor(unittest.TestCase): def test_analyze_pre_forward_without_adump(self, mock_logger_warning): self.processor.enable_kernel_dump = True self.processor.analyze_forward_input("test_api_name", None, None) - mock_logger_warning.assert_called_with("The current msprobe package does not compile adump, and kernel dump cannot be used.") + mock_logger_warning.assert_called_with( + "The current msprobe package does not compile adump, and kernel dump cannot be used.") self.assertFalse(self.processor.enable_kernel_dump) @patch('msprobe.core.data_dump.data_processor.mindspore_processor.KernelDumpDataProcessor.stop_kernel_dump') @@ -319,7 +370,8 @@ class TestKernelDumpDataProcessor(unittest.TestCase): self.processor.enable_kernel_dump = True self.processor.analyze_backward_input("test_api_name", None, None) self.assertFalse(self.processor.enable_kernel_dump) - mock_logger_warning.assert_called_with("The current msprobe package does not compile adump, and kernel dump cannot be used.") + mock_logger_warning.assert_called_with( + "The current msprobe package does not compile adump, and kernel dump cannot be used.") @patch('msprobe.core.data_dump.data_processor.mindspore_processor.KernelDumpDataProcessor.stop_kernel_dump') @patch.object(logger, 'info') diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index bda61faadf7e6ff2ca3187be75623c1f0cbbde07..ad933870c684e9862a05b2afb87250ef1789f46e 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -82,15 +82,22 @@ class TestPytorchDataProcessor(unittest.TestCase): def test_get_stat_info_float_async(self): tensor = torch.tensor([1.0, 2.0, 3.0]) - result = self.processor.get_stat_info_async(tensor).stack_tensor_stat[1] - self.assertEqual(result[0].item(), 3.0) - self.assertEqual(result[1].item(), 1.0) - self.assertEqual(result[2].item(), 2.0) - self.assertEqual(result[3].item(), torch.norm(tensor).item()) + result = self.processor.get_stat_info_async(tensor) + + result_max = result.max + result_min = result.min + result_mean = result.mean + result_norm = result.norm + + self.assertEqual(result_max.item(), 3.0) + self.assertEqual(result_min.item(), 1.0) + self.assertEqual(result_mean.item(), 2.0) + self.assertEqual(result_norm.item(), torch.norm(tensor).item()) def test_get_stat_info_int(self): tensor = torch.tensor([1, 2, 3], dtype=torch.int32) result = self.processor.get_stat_info(tensor) + self.assertEqual(result.max, 3) self.assertEqual(result.min, 1) self.assertEqual(result.mean, 2) @@ -98,11 +105,17 @@ class TestPytorchDataProcessor(unittest.TestCase): def test_get_stat_info_int_async(self): tensor = torch.tensor([1, 2, 3]) - result = self.processor.get_stat_info_async(tensor).stack_tensor_stat[1] - self.assertEqual(result[0].item(), 3.0) - self.assertEqual(result[1].item(), 1.0) - self.assertEqual(result[2].item(), 2.0) - self.assertEqual(result[3].item(), torch.norm(tensor.float()).item()) + result = self.processor.get_stat_info_async(tensor) + + result_max = result.max + result_min = result.min + result_mean = result.mean + result_norm = result.norm + + self.assertEqual(result_max.item(), 3.0) + self.assertEqual(result_min.item(), 1.0) + self.assertEqual(result_mean.item(), 2.0) + self.assertEqual(result_norm.item(), torch.norm(tensor.float()).item()) def test_get_stat_info_empty(self): tensor = torch.tensor([]) @@ -122,9 +135,13 @@ class TestPytorchDataProcessor(unittest.TestCase): def test_get_stat_info_bool_async(self): tensor = torch.tensor([True, False, True]) - result = self.processor.get_stat_info_async(tensor).stack_tensor_stat[1] - self.assertEqual(result[0].item(), True) - self.assertEqual(result[1].item(), False) + result = self.processor.get_stat_info_async(tensor) + + result_max = result.max + result_min = result.min + + self.assertEqual(result_max.item(), True) + self.assertEqual(result_min.item(), False) def test_get_stat_info_with_scalar_tensor(self): scalar_tensor = torch.tensor(42.0) @@ -206,7 +223,7 @@ class TestPytorchDataProcessor(unittest.TestCase): dist.init_process_group(backend='gloo', world_size=1, rank=0) process_group_element = dist.group.WORLD result = self.processor.process_group_hash(process_group_element) - expected = hashlib.md5('[0]'.encode('utf-8')).hexdigest() + expected = f"{zlib.crc32(str([0]).encode('utf-8')):08x}" self.assertEqual(result, expected) def test_analyze_torch_size(self): @@ -232,7 +249,7 @@ class TestPytorchDataProcessor(unittest.TestCase): expected = { 'type': 'torch.ProcessGroup', 'group_ranks': [0], - 'group_id': hashlib.md5('[0]'.encode('utf-8')).hexdigest() + 'group_id': f"{zlib.crc32(str([0]).encode('utf-8')):08x}" } self.assertEqual(result, expected) @@ -247,6 +264,7 @@ class TestPytorchDataProcessor(unittest.TestCase): class TestReduceOp: def __str__(self): raise Exception("failed to convert str type") + arg = TestReduceOp() self.processor._analyze_reduce_op(arg) mock_logger_warning.assert_called_with( @@ -278,11 +296,35 @@ class TestPytorchDataProcessor(unittest.TestCase): self.assertEqual(result, self.processor._analyze_process_group(process_group_element)) def test_analyze_single_element_numpy_conversion(self): - numpy_element = np.int64(1) - converted_numpy, numpy_type = self.processor._convert_numpy_to_builtin(numpy_element) + numpy_element = np.int32(5) result = self.processor.analyze_single_element(numpy_element, []) - expected_result = {"type": numpy_type, "value": converted_numpy} - self.assertEqual(result, expected_result) + expected = {"type": 'int32', "value": 5} + self.assertEqual(result, expected) + + numpy_element = np.float32(3.14) + result = self.processor.analyze_single_element(numpy_element, []) + expected = {"type": 'float32', "value": 3.140000104904175} + self.assertEqual(result, expected) + + numpy_element = np.bool_(True) + result = self.processor.analyze_single_element(numpy_element, []) + expected = {"type": 'bool_', "value": True} + self.assertEqual(result, expected) + + numpy_element = np.str_("abc") + result = self.processor.analyze_single_element(numpy_element, []) + expected = {"type": 'str_', "value": "abc"} + self.assertEqual(result, expected) + + numpy_element = np.byte(1) + result = self.processor.analyze_single_element(numpy_element, []) + expected = {"type": 'int8', "value": 1} + self.assertEqual(result, expected) + + numpy_element = np.complex128(1 + 2j) + result = self.processor.analyze_single_element(numpy_element, []) + expected = {"type": 'complex128', "value": (1 + 2j)} + self.assertEqual(result, expected) def test_analyze_single_element_tensor(self): tensor_element = torch.tensor([1, 2, 3]) @@ -312,28 +354,20 @@ class TestPytorchDataProcessor(unittest.TestCase): 'type': 'torch.Tensor', 'dtype': str(tensor.dtype), 'shape': tensor.shape, - 'Max': 3.0, - 'Min': 1.0, - 'Mean': 2.0, - 'Norm': torch.norm(tensor).item(), 'requires_grad': tensor.requires_grad, 'md5': 'mocked_md5' } + result.pop('tensor_stat_index', None) self.assertDictEqual(expected, result) def test_analyze_tensor_with_empty_tensor(self): tensor = torch.tensor([]) result = self.processor._analyze_tensor(tensor, 'suffix') - self.assertEqual(result['Max'], None) - self.assertEqual(result['Min'], None) - self.assertEqual(result['Mean'], None) - self.assertEqual(result['Norm'], None) - def test_analyze_tensor_with_inf_and_nan(self): - tensor = torch.tensor([1.0, float('inf'), float('nan'), -float('inf')]) - result = self.processor._analyze_tensor(tensor, 'suffix') - self.assertEqual(result['Max_except_inf_nan'], 1.0) - self.assertEqual(result['Min_except_inf_nan'], 1.0) + self.assertEqual(result['type'], "torch.Tensor") + self.assertEqual(result['dtype'], 'torch.float32') + self.assertEqual(result['shape'], torch.Size([0])) + self.assertEqual(result['requires_grad'], False) def test_cast_to_float_if_fp8(self): tensor = MagicMock() @@ -368,13 +402,10 @@ class TestTensorDataProcessor(unittest.TestCase): 'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': tensor.shape, - 'Max': 3.0, - 'Min': 1.0, - 'Mean': 2.0, - 'Norm': torch.norm(tensor).item(), 'requires_grad': False, 'data_name': 'test_api.input.suffix.pt' } + result.pop('tensor_stat_index', None) self.assertEqual(expected, result) @@ -392,6 +423,9 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): sys.modules['torch_npu'] = Mock() sys.modules['torch_npu.npu'] = Mock() sys.modules['torch_npu.npu.utils'] = Mock() + self.tensor_json = { + 'tensor_stat_index': 123 # 默认情况下 tensor_stat_index 存在 + } def test_is_terminated(self): self.processor.overflow_nums = -1 @@ -406,7 +440,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): def test_analyze_forward_input(self): with patch.object(BaseDataProcessor, "analyze_forward_input", return_value={"name": 1}): - api_info = self.processor.analyze_forward_input("name", "module","module_input_output") + api_info = self.processor.analyze_forward_input("name", "module", "module_input_output") self.assertEqual(self.processor.cached_api_info, {"name": 1}) self.assertIsNone(api_info) @@ -468,19 +502,43 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.processor._is_support_inf_nan() self.assertTrue(self.processor.support_inf_nan) - def test_analyze_maybe_overflow_tensor(self): - tensor_json = {'Max': None, 'Min': None} - self.processor._analyze_maybe_overflow_tensor(tensor_json) + def test_max_tensor_or_min_tensor_is_none(self): + # 让 get_buffer_values_max 和 get_buffer_values_min 返回 None + self.processor.data_writer.get_buffer_values_max.return_value = None + self.processor.data_writer.get_buffer_values_min.return_value = None + + # 在该情况下应该直接返回，不做任何改变 + self.processor._analyze_maybe_overflow_tensor(self.tensor_json) + + # 确保 has_overflow 没有被设置 self.assertFalse(self.processor.has_overflow) - tensor_json = {'Max': float('inf'), 'Min': 1.0} - self.processor._analyze_maybe_overflow_tensor(tensor_json) + def test_tensor_is_inf_or_nan(self): + # 模拟 max_tensor 为 Inf + self.processor.data_writer.get_buffer_values_max.return_value = torch.tensor(float('inf')) + self.processor.data_writer.get_buffer_values_min.return_value = torch.tensor(1.0) + + # 测试应该设置 has_overflow 为 True + self.processor._analyze_maybe_overflow_tensor(self.tensor_json) self.assertTrue(self.processor.has_overflow) - tensor_json = {'Max': 1.0, 'Min': float('inf')} - self.processor._analyze_maybe_overflow_tensor(tensor_json) + # 模拟 min_tensor 为 NaN + self.processor.data_writer.get_buffer_values_max.return_value = torch.tensor(1.0) + self.processor.data_writer.get_buffer_values_min.return_value = torch.tensor(float('nan')) + + # 测试应该设置 has_overflow 为 True + self.processor._analyze_maybe_overflow_tensor(self.tensor_json) self.assertTrue(self.processor.has_overflow) + def test_normal_tensor(self): + # 模拟正常的 max_tensor 和 min_tensor + self.processor.data_writer.get_buffer_values_max.return_value = torch.tensor(1.0) + self.processor.data_writer.get_buffer_values_min.return_value = torch.tensor(-1.0) + + # 在正常情况下不应该改变 has_overflow + self.processor._analyze_maybe_overflow_tensor(self.tensor_json) + self.assertFalse(self.processor.has_overflow) + @patch('msprobe.core.common.file_utils.path_len_exceeds_limit', return_value=False) @patch.object(BaseDataProcessor, 'get_save_file_path', return_value=['test_api_name', 'test_api_name.0.forward.input.pt']) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index b9d2e7abef7244fc12dc71e3113c26af52529ce9..6be25453016774b3e33787e6da8b5b17e5e5c868 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # @@ -13,8 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" +import os import unittest from unittest.mock import patch, mock_open, MagicMock @@ -22,9 +21,6 @@ from msprobe.core.common.utils import Const from msprobe.core.data_dump.data_collector import DataCollector from msprobe.pytorch.debugger.debugger_config import DebuggerConfig from msprobe.pytorch.pt_config import parse_json_config -from msprobe.core.data_dump.json_writer import DataWriter -from msprobe.core.data_dump.data_processor.base import BaseDataProcessor -from msprobe.core.data_dump.data_processor.pytorch_processor import StatisticsDataProcessor class TestDataCollector(unittest.TestCase): @@ -38,6 +34,143 @@ class TestDataCollector(unittest.TestCase): config = DebuggerConfig(common_config, task_config, Const.STATISTICS, "./ut_dump", "L1") self.data_collector = DataCollector(config) + def test_dump_data_dir(self): + self.assertEqual(self.data_collector.dump_data_dir, None) + + self.data_collector.data_writer.dump_tensor_data_dir = "./test_dump" + self.assertEqual(self.data_collector.dump_data_dir, "./test_dump") + + def test_dump_file_path(self): + self.assertEqual(self.data_collector.dump_file_path, None) + + self.data_collector.data_writer.dump_file_path = "./test_dump/dump.json" + self.assertEqual(self.data_collector.dump_file_path, "./test_dump/dump.json") + + def test_scope_none_and_pid_match(self): + mock_name = "test_module" + current_pid = os.getpid() + result = self.data_collector.check_scope_and_pid(None, mock_name, current_pid) + self.assertTrue(result) + + def test_scope_valid_and_pid_match(self): + mock_scope = MagicMock() + mock_scope.check.return_value = True + mock_name = "valid_module" + current_pid = os.getpid() + result = self.data_collector.check_scope_and_pid(mock_scope, mock_name, current_pid) + self.assertTrue(result) + mock_scope.check.assert_called_once_with(mock_name) + + def test_scope_invalid_and_pid_match(self): + mock_scope = MagicMock() + mock_scope.check.return_value = False + mock_name = "invalid_module" + current_pid = os.getpid() + result = self.data_collector.check_scope_and_pid(mock_scope, mock_name, current_pid) + self.assertFalse(result) + + def test_scope_valid_but_pid_mismatch(self): + mock_scope = MagicMock() + mock_scope.check.return_value = True + mock_name = "valid_module" + fake_pid = os.getpid() + 1 + result = self.data_collector.check_scope_and_pid(mock_scope, mock_name, fake_pid) + self.assertFalse(result) + + def test_scope_none_but_pid_mismatch(self): + mock_name = "test_module" + fake_pid = os.getpid() + 1 + result = self.data_collector.check_scope_and_pid(None, mock_name, fake_pid) + self.assertFalse(result) + + def test_normal_case(self): + data_info = {"key1": {"other_field": "value"}} + self.data_collector.set_is_recomputable(data_info, True) + self.assertTrue(data_info["key1"]["is_recompute"]) + + self.data_collector.set_is_recomputable(data_info, False) + self.assertFalse(data_info["key1"]["is_recompute"]) + + def test_empty_data_info(self): + data_info = {} + original_data = data_info.copy() + self.data_collector.set_is_recomputable(data_info, True) + self.assertEqual(data_info, original_data) + + def test_data_info_length_not_one(self): + data_info = {"key1": {}, "key2": {}} + original_data = data_info.copy() + self.data_collector.set_is_recomputable(data_info, True) + self.assertEqual(data_info, original_data) + + def test_is_recompute_none(self): + data_info = {"key1": {}} + original_data = data_info.copy() + self.data_collector.set_is_recomputable(data_info, None) + self.assertEqual(data_info, original_data) + + def test_nested_structure(self): + data_info = {"layer1": {"sub_layer": {"value": 1}}} + self.data_collector.set_is_recomputable(data_info, True) + self.assertTrue(data_info["layer1"]["is_recompute"]) + self.assertEqual(data_info["layer1"]["sub_layer"]["value"], 1) + + def test_reset_status(self): + self.data_collector.optimizer_status = "test_optimizer_status" + self.data_collector.reset_status() + + self.assertEqual(self.data_collector.optimizer_status, "") + self.assertEqual( + self.data_collector.optimizer_status_first_start, + {Const.OPTIMIZER: True, Const.CLIP_GRAD: True} + ) + self.assertEqual(self.data_collector.backward_module_names, {}) + + def test_update_api_or_module_name(self): + self.assertEqual(self.data_collector.data_processor.current_api_or_module_name, None) + + self.data_collector.update_api_or_module_name("test_api_name") + self.assertEqual(self.data_collector.data_processor.current_api_or_module_name, "test_api_name") + + def test_write_json(self): + self.data_collector.data_writer = MagicMock() + + self.data_collector.write_json() + self.data_collector.data_writer.write_json.assert_called_once() + + def test_write_json_at_exit_with_async_dump_tensor(self): + self.data_collector.data_processor = MagicMock() + self.data_collector.data_writer = MagicMock() + self.data_collector.config.async_dump = True + self.data_collector.config.task = "tensor" + + self.data_collector.write_json_at_exit() + + self.data_collector.data_processor.dump_async_data.assert_called_once() + self.data_collector.data_writer.write_json.assert_called_once() + + def test_write_json_at_exit_with_no_async_dump(self): + self.data_collector.data_processor = MagicMock() + self.data_collector.data_writer = MagicMock() + self.data_collector.config.async_dump = False + self.data_collector.config.task = "tensor" + + self.data_collector.write_json_at_exit() + + self.data_collector.data_processor.dump_async_data.assert_not_called() + self.data_collector.data_writer.write_json.assert_called_once() + + def test_write_json_at_exit_with_statistics(self): + self.data_collector.data_processor = MagicMock() + self.data_collector.data_writer = MagicMock() + self.data_collector.config.async_dump = True + self.data_collector.config.task = "statistics" + + self.data_collector.write_json_at_exit() + + self.data_collector.data_processor.dump_async_data.assert_not_called() + self.data_collector.data_writer.write_json.assert_called_once() + def test_update_data(self): self.data_collector.config.task = Const.OVERFLOW_CHECK self.data_collector.data_processor.has_overflow = True @@ -59,6 +192,82 @@ class TestDataCollector(unittest.TestCase): mock_warning.assert_not_called() mock_debug.assert_called_once_with("msprobe is collecting data on Tensor.add.") + def test_call_stack_collect(self): + self.data_collector.data_processor = MagicMock() + self.data_collector.data_writer = MagicMock() + + test_name = "test_api" + mock_stack = ["func1", "func2", "func3"] + self.data_collector.data_processor.analyze_api_call_stack.return_value = mock_stack + + self.data_collector.call_stack_collect(test_name) + + self.data_collector.data_processor.analyze_api_call_stack.assert_called_once_with(test_name) + self.data_collector.data_writer.update_stack.assert_called_once_with(test_name, mock_stack) + + def test_update_construct_without_construct(self): + self.data_collector.data_writer = MagicMock() + + self.data_collector.config.level = "L1" + self.data_collector.update_construct("test") + self.data_collector.data_writer.update_construct.assert_not_called() + + def test_update_construct_with_first_start(self): + self.data_collector.module_processor = MagicMock() + self.data_collector.data_writer = MagicMock() + self.data_collector.config.level = "L0" + self.data_collector.optimizer_status = "optimizer" + self.data_collector.optimizer_status_first_start = {"optimizer": True} + + self.data_collector.update_construct("test_name") + calls = [ + unittest.mock.call({"optimizer": None}), + unittest.mock.call({"test_name": "optimizer"}), + unittest.mock.call(self.data_collector.module_processor.module_node) + ] + self.data_collector.data_writer.update_construct.assert_has_calls(calls) + + def test_update_construct_with_not_first_start(self): + self.data_collector.module_processor = MagicMock() + self.data_collector.data_writer = MagicMock() + self.data_collector.config.level = "L0" + self.data_collector.optimizer_status = "clip_grad" + self.data_collector.optimizer_status_first_start = {"clip_grad": False} + + self.data_collector.update_construct("test_name") + calls = [ + unittest.mock.call({"test_name": "clip_grad"}), + unittest.mock.call(self.data_collector.module_processor.module_node) + ] + self.data_collector.data_writer.update_construct.assert_has_calls(calls) + + def test_update_construct_with_module_prefix(self): + self.data_collector.module_processor = MagicMock() + self.data_collector.data_writer = MagicMock() + self.data_collector.config.level = "mix" + self.data_collector.optimizer_status = "other_status" + test_name = "Module_test_name" + + self.data_collector.update_construct(test_name) + self.data_collector.data_writer.update_construct.assert_called_with( + self.data_collector.module_processor.module_node + ) + + def test_update_construct_without_module_prefix(self): + self.data_collector.module_processor = MagicMock() + self.data_collector.data_writer = MagicMock() + self.data_collector.config.level = "mix" + self.data_collector.optimizer_status = "other_status" + self.data_collector.module_processor.api_parent_node = "parent_node" + test_name = "api_name" + + self.data_collector.update_construct(test_name) + calls = [ + unittest.mock.call({"api_name": "parent_node"}), + unittest.mock.call(self.data_collector.module_processor.module_node) + ] + self.data_collector.data_writer.update_construct.assert_has_calls(calls) + def test_handle_data(self): with patch.object(DataCollector, "update_data") as mock_update_data, \ patch.object(DataCollector, "write_json") as mock_write_json, \ @@ -76,44 +285,213 @@ class TestDataCollector(unittest.TestCase): mock_flush.assert_not_called() mock_write_json.assert_called() - @patch.object(DataCollector, "update_construct") - @patch.object(DataWriter, "update_stack") - @patch.object(BaseDataProcessor, "analyze_api_call_stack") - @patch.object(DataCollector, "handle_data") - def test_forward_data_collect(self, mock_handle_data, _, __, ___): - with patch.object(DataCollector, "check_scope_and_pid", return_value=True), \ - patch.object(StatisticsDataProcessor, "analyze_forward", return_value={}): - with patch.object(StatisticsDataProcessor, "is_terminated", new=True): - self.data_collector.forward_data_collect("name", "module", "pid", "module_input_output") - mock_handle_data.assert_called_with("name", {}, flush=True) - - self.data_collector.forward_data_collect("name", "module", "pid", "module_input_output") - mock_handle_data.assert_called_with("name", {}, flush=False) - - @patch.object(DataCollector, "update_construct") - @patch.object(DataCollector, "handle_data") - def test_backward_data_collect(self, mock_handle_data, _): - with patch.object(DataCollector, "check_scope_and_pid", return_value=True), \ - patch.object(StatisticsDataProcessor, "analyze_backward", return_value={}): - with patch.object(StatisticsDataProcessor, "is_terminated", new=True): - self.data_collector.backward_data_collect("name", "module", "pid", "module_input_output") - mock_handle_data.assert_called_with("name", {}, flush=True) - - self.data_collector.backward_data_collect("name", "module", "pid", "module_input_output") - mock_handle_data.assert_called_with("name", {}, flush=False) - - @patch.object(DataWriter, "update_debug") - @patch.object(BaseDataProcessor, "analyze_debug_forward", return_value="data_info") - def test_debug_data_collect_forward(self, _, mock_update_debug): - self.data_collector.debug_data_collect_forward("variable", "name_with_count") - mock_update_debug.assert_called_with({"name_with_count": "data_info"}) - - @patch.object(DataWriter, "update_debug") - @patch.object(BaseDataProcessor, "analyze_debug_backward") - @patch.object(BaseDataProcessor, "analyze_element_to_all_none", return_value = "all_none_data_info") - def test_debug_data_collect_backward(self, _, mock_analyze_debug_backward, mock_update_debug): - self.data_collector.data_writer.cache_debug = {"data": None} - self.data_collector.debug_data_collect_backward("variable", "name_with_count") - mock_update_debug.assert_called_with({"name_with_count": "all_none_data_info"}) - mock_analyze_debug_backward.assert_called_with("variable", "name_with_count", self.data_collector.data_writer.cache_debug['data']) - self.data_collector.data_writer.cache_debug = None + +class TestForwardDataCollect(unittest.TestCase): + def setUp(self): + mock_json_data = { + "dump_path": "./test_fwd_dump", + } + with patch("msprobe.pytorch.pt_config.FileOpen", mock_open(read_data='')), \ + patch("msprobe.pytorch.pt_config.load_json", return_value=mock_json_data): + common_config, task_config = parse_json_config("./config.json", Const.STATISTICS) + config = DebuggerConfig(common_config, task_config, Const.STATISTICS, "./test_fwd_dump", "L1") + self.data_collector = DataCollector(config) + + self.data_collector.update_construct = MagicMock() + self.data_collector.config = MagicMock() + self.data_collector.data_processor = MagicMock() + self.data_collector.scope = "test_scope" + self.data_collector.check_scope_and_pid = MagicMock() + self.data_collector.set_is_recomputable = MagicMock() + self.data_collector.handle_data = MagicMock() + self.data_collector.call_stack_collect = MagicMock() + + self.Const = MagicMock() + self.Const.FREE_BENCHMARK = "free_benchmark" + self.Const.TENSOR = "tensor" + self.Const.FORWARD = "forward" + self.Const.BACKWARD = "backward" + self.Const.STRUCTURE = "structure" + self.Const.LEVEL_L2 = "L2" + + def test_forward_input_with_free_benchmark_task(self): + self.data_collector.config.task = self.Const.FREE_BENCHMARK + self.data_collector.check_scope_and_pid.return_value = True + + self.data_collector.forward_input_data_collect( + "forward_test", + "module1", + 123, + "input_output" + ) + + self.data_collector.data_processor.analyze_forward_input.assert_called_once_with( + "backward_test", + "module1", + "input_output" + ) + + def test_forward_input_with_scope_pid_check_fail(self): + self.data_collector.config.task = self.Const.TENSOR + self.data_collector.check_scope_and_pid.return_value = False + + self.data_collector.forward_input_data_collect( + "test", "module1", 123, "input_output" + ) + + self.data_collector.data_processor.analyze_forward_input.assert_not_called() + + def test_forward_input_with_structure_task(self): + self.data_collector.config.task = self.Const.STRUCTURE + self.data_collector.check_scope_and_pid.return_value = True + + self.data_collector.forward_input_data_collect( + "test", "module1", 123, "input_output" + ) + + self.data_collector.data_processor.analyze_forward_input.assert_not_called() + self.data_collector.set_is_recomputable.assert_called_once_with({}, None) + + def test_forward_input_with_level_l2(self): + self.data_collector.config.task = self.Const.TENSOR + self.data_collector.config.level = self.Const.LEVEL_L2 + self.data_collector.check_scope_and_pid.return_value = True + + self.data_collector.forward_input_data_collect( + "test", "module1", 123, "input_output" + ) + + self.data_collector.handle_data.assert_not_called() + + def test_forward_input_with_recompute(self): + self.data_collector.config.task = self.Const.TENSOR + self.data_collector.config.level = "L1" + self.data_collector.check_scope_and_pid.return_value = True + mock_data = {"key": "value"} + self.data_collector.data_processor.analyze_forward_input.return_value = mock_data + + self.data_collector.forward_input_data_collect( + "test", "module1", 123, "input_output", is_recompute=True + ) + + self.data_collector.set_is_recomputable.assert_called_once_with(mock_data, True) + self.data_collector.handle_data.assert_called_once_with( + "test", mock_data, flush=self.data_collector.data_processor.is_terminated + ) + + def test_forward_output_with_scope_check_fail(self): + self.data_collector.check_scope_and_pid.return_value = False + self.data_collector.forward_output_data_collect("test", "module", 123, "data") + self.data_collector.data_processor.analyze_forward_output.assert_not_called() + + def test_forward_output_with_structure_task(self): + self.data_collector.config.task = self.Const.STRUCTURE + self.data_collector.forward_output_data_collect("test", "module", 123, "data") + self.data_collector.data_processor.analyze_forward_output.assert_not_called() + + def test_forward_output_with_level_l2(self): + self.data_collector.config.level = self.Const.LEVEL_L2 + self.data_collector.forward_output_data_collect("test", "module", 123, "data") + self.data_collector.handle_data.assert_not_called() + + def test_forward_output_normal(self): + mock_data = {"key": "value"} + self.data_collector.data_processor.analyze_forward_output.return_value = mock_data + self.data_collector.forward_output_data_collect("test", "module", 123, "data", True) + self.data_collector.call_stack_collect.assert_called_once_with("test") + self.data_collector.handle_data.assert_called_once_with( + "test", + mock_data, + flush=self.data_collector.data_processor.is_terminated + ) + + def test_forward_with_scope_check_fail(self): + self.data_collector.check_scope_and_pid.return_value = False + self.data_collector.forward_data_collect("test", "module", 123, "data") + self.data_collector.data_processor.analyze_forward.assert_not_called() + + def test_forward_with_structure_task(self): + self.data_collector.config.task = self.Const.STRUCTURE + self.data_collector.forward_data_collect("test", "module", 123, "data") + self.data_collector.data_processor.analyze_forward.assert_not_called() + + def test_forward_normal(self): + mock_data = {"key": "value"} + self.data_collector.data_processor.analyze_forward.return_value = mock_data + self.data_collector.forward_data_collect("test", "module", 123, "data", False) + self.data_collector.call_stack_collect.assert_called_once_with("test") + self.data_collector.handle_data.assert_called_once_with( + "test", + mock_data, + flush=self.data_collector.data_processor.is_terminated + ) + + +class TestBackwardDataCollector(unittest.TestCase): + def setUp(self): + mock_json_data = { + "dump_path": "./test_bwd_dump", + } + with patch("msprobe.pytorch.pt_config.FileOpen", mock_open(read_data='')), \ + patch("msprobe.pytorch.pt_config.load_json", return_value=mock_json_data): + common_config, task_config = parse_json_config("./config.json", Const.STATISTICS) + config = DebuggerConfig(common_config, task_config, Const.STATISTICS, "./test_bwd_dump", "L1") + self.data_collector = DataCollector(config) + + self.data_collector.config = MagicMock() + self.data_collector.data_processor = MagicMock() + self.data_collector.scope = "test_scope" + self.data_collector.check_scope_and_pid = MagicMock(return_value=True) + self.data_collector.set_is_recomputable = MagicMock() + self.data_collector.handle_data = MagicMock() + self.data_collector.update_construct = MagicMock() + self.data_collector.backward_module_names = {} + + self.Const = MagicMock() + self.Const.STRUCTURE = "structure" + self.Const.TENSOR = "tensor" + self.Const.LEVEL_L2 = "L2" + self.Const.SEP = "." + self.Const.MODULE_PREFIX = ["module"] + + def test_backward_with_scope_check_fail(self): + self.data_collector.check_scope_and_pid.return_value = False + self.data_collector.backward_data_collect("test", "module", 123, "data") + self.data_collector.data_processor.analyze_backward.assert_not_called() + + def test_backward_with_level_l2(self): + self.data_collector.config.level = self.Const.LEVEL_L2 + self.data_collector.backward_data_collect("test", "module", 123, "data") + self.data_collector.handle_data.assert_not_called() + + def test_backward_data_module_prefix_match(self): + self.data_collector.check_scope_and_pid.return_value = True + self.data_collector.config.task = self.Const.TENSOR + self.data_collector.config.level = "L1" + mock_data = {"key": "value"} + self.data_collector.data_processor.analyze_backward.return_value = mock_data + test_name = "Module.layer1.backward" + self.data_collector.backward_data_collect(test_name, "module", 123, "data") + self.assertEqual(self.data_collector.backward_module_names, {"Module": True}) + + def test_backward_input_with_structure_task(self): + self.data_collector.config.task = self.Const.STRUCTURE + self.data_collector.backward_input_data_collect("test", "module", 123, "data") + self.data_collector.data_processor.analyze_backward_input.assert_not_called() + + def test_backward_input_with_normal(self): + mock_data = {"key": "value"} + self.data_collector.data_processor.analyze_backward_input.return_value = mock_data + self.data_collector.backward_input_data_collect("test", "module", 123, "data", True) + self.data_collector.set_is_recomputable.assert_called_once_with(mock_data, True) + + def test_backward_output_with_scope_check_fail(self): + self.data_collector.check_scope_and_pid.return_value = False + self.data_collector.backward_output_data_collect("test", "module", 123, "data") + self.data_collector.data_processor.analyze_backward_output.assert_not_called() + + def test_backward_output_with_recompute(self): + mock_data = {"key": "value"} + self.data_collector.data_processor.analyze_backward_output.return_value = mock_data + self.data_collector.backward_output_data_collect("test", "module", 123, "data", False) + self.data_collector.set_is_recomputable.assert_called_once_with(mock_data, False) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py index 9b20ffb2197882e16c1550cf013d1ba132096063..e9eda0a9a1e80352974b5fc27820d902ca9feff2 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py @@ -3,6 +3,7 @@ import os import unittest from unittest.mock import patch +from msprobe.core.common.const import Const from msprobe.core.common.utils import DumpPathAggregation from msprobe.core.common.file_utils import FileOpen, remove_path, load_json from msprobe.core.data_dump.json_writer import DataWriter @@ -13,6 +14,49 @@ class TestDataWriter(unittest.TestCase): self.data_writer = DataWriter() self.data_content = {"task": "tensor", "level": "L1", "data": {"Tensor.add": 1}} self.cur_path = os.path.dirname(os.path.realpath(__file__)) + self.stat_vector = [1.0, 2.0, 3.0, 4.0] # Example stat_vector for tests + self.data_writer.stat_stack_list = [self.stat_vector] # Mock the stat_stack_list + + def test_replace_stat_placeholders(self): + stat_result = [[1.0, 2.0, 3.0, 4.0]] # Mocking stat_result with a dummy value + data = {"type": "Tensor", "dtype": "float32", "shape": [1, 2, 3], Const.TENSOR_STAT_INDEX: 0} + + # Call _replace_stat_placeholders directly + self.data_writer._replace_stat_placeholders(data, stat_result) + + # Check that the function processed the placeholders correctly + self.assertEqual(data["Max"], 1.0) + self.assertEqual(data["Min"], 2.0) + self.assertEqual(data["Mean"], 3.0) + self.assertEqual(data["Norm"], 4.0) + + def test_append_stat_to_buffer(self): + index = self.data_writer.append_stat_to_buffer(self.stat_vector) + self.assertEqual(index, 1) # The first append will return index 0 + self.assertEqual(self.data_writer.stat_stack_list[0], + self.stat_vector) # Check if the stat is appended correctly + + def test_get_buffer_values_max(self): + max_value = self.data_writer.get_buffer_values_max(0) + self.assertEqual(max_value, 1.0) # The max value of stat_vector is 1.0 + + # Test when index is out of range + max_value_invalid = self.data_writer.get_buffer_values_max(1) + self.assertIsNone(max_value_invalid) # Should return None for invalid index + + def test_get_buffer_values_min(self): + min_value = self.data_writer.get_buffer_values_min(0) + self.assertEqual(min_value, 2.0) # The min value of stat_vector is 2.0 + + # Test when index is out of range + min_value_invalid = self.data_writer.get_buffer_values_min(1) + self.assertIsNone(min_value_invalid) # Should return None for invalid index + + def test_flush_stat_stack(self): + # Ensure that flush_stat_stack works and clears the stat_stack_list + result = self.data_writer.flush_stat_stack() + self.assertEqual(result, [[1.0, 2.0, 3.0, 4.0]]) # Returns the flushed stats + self.assertEqual(self.data_writer.stat_stack_list, []) # Ensure the list is cleared after flush def test_write_data_to_csv(self): cur_path = os.path.dirname(os.path.realpath(__file__)) @@ -42,9 +86,9 @@ class TestDataWriter(unittest.TestCase): remove_path(file_path) def test_reset_cache(self): - self.data_writer.cache_data={"data": 1} - self.data_writer.cache_stack={"stack": 2} - self.data_writer.cache_construct={"construct": 3} + self.data_writer.cache_data = {"data": 1} + self.data_writer.cache_stack = {"stack": 2} + self.data_writer.cache_construct = {"construct": 3} self.data_writer.reset_cache() self.assertEqual(self.data_writer.cache_data, {}) self.assertEqual(self.data_writer.cache_stack, {}) @@ -117,8 +161,9 @@ class TestDataWriter(unittest.TestCase): self.assertEqual(self.data_writer.cache_data, expected) def test_update_stack(self): - self.data_writer.update_stack(self.data_content) - self.assertEqual(self.data_writer.cache_stack, self.data_content) + self.data_writer.cache_stack = {"stack1": ["test1"]} + self.data_writer.update_stack("test2", "stack1") + self.assertEqual(self.data_writer.cache_stack, {"stack1": ["test1", "test2"]}) def test_update_construct(self): self.data_writer.update_construct(self.data_content) @@ -136,13 +181,13 @@ class TestDataWriter(unittest.TestCase): os.remove(file_path) def test_write_stack_info_json(self): - self.data_writer.cache_stack = self.data_content + self.data_writer.cache_stack = {("api1", "api2"): ["stack1"]} file_path = os.path.join(self.cur_path, "stack.json") self.data_writer.write_stack_info_json(file_path) load_result = load_json(file_path) try: - self.assertEqual(load_result, self.data_content) + self.assertEqual(load_result, {"0": [["stack1"], ["api1", "api2"]]}) finally: os.remove(file_path) @@ -156,3 +201,48 @@ class TestDataWriter(unittest.TestCase): self.assertEqual(load_result, self.data_content) finally: os.remove(file_path) + + def test_replace_stat_placeholders_invalid_index(self): + data = { + "type": "Tensor", + "dtype": "float32", + "shape": [1, 2], + Const.TENSOR_STAT_INDEX: 10 # 超出索引 + } + stat_result = [[1.0, 2.0, 3.0, 4.0]] + self.data_writer._replace_stat_placeholders(data, stat_result) + self.assertIsNone(data.get(Const.TENSOR_STAT_INDEX)) + self.assertIn(Const.MAX, data) + self.assertIsNone(data[Const.MAX]) # 越界填 None + + def test_append_stat_to_buffer_multiple(self): + for i in range(5): + idx = self.data_writer.append_stat_to_buffer([i, i+1, i+2, i+3]) + self.assertEqual(idx, i + 1) + self.assertEqual(len(self.data_writer.stat_stack_list), 6) # 包含 setUp 中那一条 + + def test_get_buffer_values_max_invalid_data(self): + self.data_writer.stat_stack_list = [["not-a-number"]] # 非预期格式 + max_val = self.data_writer.get_buffer_values_max(0) + self.assertEqual(max_val, "not-a-number") # 仍然返回第一位 + + max_val = self.data_writer.get_buffer_values_max(-1) + self.assertIsNone(max_val) + + def test_flush_stat_stack_empty(self): + self.data_writer.stat_stack_list = [] + result = self.data_writer.flush_stat_stack() + self.assertEqual(result, []) + + def test_flush_stat_stack_with_tensor_like_items(self): + class DummyTensor: + def __init__(self, v): self.v = v + def item(self): return self.v + + self.data_writer.stat_stack_list = [ + [DummyTensor(1), DummyTensor(2), DummyTensor(3), DummyTensor(4)], + [5.5, 6.6, 7.7, 8.8] # 混合类型 + ] + result = self.data_writer.flush_stat_stack() + self.assertEqual(result, [[1, 2, 3, 4], [5.5, 6.6, 7.7, 8.8]]) + self.assertEqual(self.data_writer.stat_stack_list, []) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/single_save/test_single_save.py b/debug/accuracy_tools/msprobe/test/core_ut/single_save/test_single_save.py new file mode 100644 index 0000000000000000000000000000000000000000..95110f7878adea55d6eec6a559a1e5ead1ce9b11 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/single_save/test_single_save.py @@ -0,0 +1,106 @@ +import unittest +import os +import shutil +import torch +import torch.nn as nn +import mindspore +import mindspore.nn as mnn +from mindspore import Tensor +from msprobe.core import SingleSave +from msprobe.core import SingleComparator +from msprobe.core.common.file_utils import read_xlsx + + +# 固定随机性 +torch.manual_seed(42) +mindspore.set_seed(42) + + +# 定义 PyTorch 简单网络 +class SimpleTorchNet(nn.Module): + def __init__(self): + super(SimpleTorchNet, self).__init__() + self.fc1 = nn.Linear(10, 5) + self.fc2 = nn.Linear(5, 1) + + def forward(self, x): + x = self.fc1(x) + x = torch.relu(x) + output = self.fc2(x) + return x, output + + +# 定义 MindSpore 简单网络 +class SimpleMindSporeNet(mnn.Cell): + def __init__(self): + super(SimpleMindSporeNet, self).__init__() + self.fc1 = mnn.Dense(10, 5) + self.fc2 = mnn.Dense(5, 1) + + def construct(self, x): + x = self.fc1(x) + x = mindspore.ops.relu(x) + output = self.fc2(x) + return x, output + + +class TestNetworkComparison(unittest.TestCase): + def setUp(self): + self.torch_dump_path = "./torch_dump" + self.mindspore_dump_path = "./mindspore_dump" + self.output_path = "./compare_output" + self.num_test_cases = 5 # 随机测试用例数量 + + def tearDown(self): + if os.path.exists(self.torch_dump_path): + shutil.rmtree(self.torch_dump_path) + if os.path.exists(self.mindspore_dump_path): + shutil.rmtree(self.mindspore_dump_path) + if os.path.exists(self.output_path): + shutil.rmtree(self.output_path) + + def run_torch_network(self): + net = SimpleTorchNet() + saver = SingleSave(self.torch_dump_path, fmk="pytorch") + + for i in range(self.num_test_cases): + # 为每个测试用例生成不同的随机输入 + input_tensor = torch.randn(1, 10) + x, output = net(input_tensor) + saver.save({"output1": x, "output2": output}) + saver.save({"output1": x}) + saver.step() # 每个输入对应一个step + + def run_mindspore_network(self): + net = SimpleMindSporeNet() + SingleSave._instance = None # 重置单例 + saver = SingleSave(self.mindspore_dump_path, fmk="mindspore") + + for i in range(self.num_test_cases): + # 为每个测试用例生成不同的随机输入 + input_tensor = Tensor(mindspore.numpy.randn(1, 10)) + x, output = net(input_tensor) + saver.save({"output1": x, "output2": output}) + saver.save({"output1": x}) + saver.step() # 每个输入对应一个step + + def test_network_comparison(self): + # 运行 PyTorch 网络并保存多组数据 + self.run_torch_network() + + # 运行 MindSpore 网络并保存多组数据 + self.run_mindspore_network() + + # 使用 SingleComparator 进行对比 + SingleComparator.compare(self.torch_dump_path, self.mindspore_dump_path, self.output_path) + + # 验证输出目录是否存在 + self.assertTrue(os.path.exists(self.output_path)) + + output1_xlsx = read_xlsx(os.path.join(self.output_path, "output1.xlsx")) + self.assertEqual(output1_xlsx.columns.tolist(), SingleComparator.result_header) + self.assertEqual(len(output1_xlsx), 10) + + output2_xlsx = read_xlsx(os.path.join(self.output_path, "output2.xlsx")) + self.assertEqual(output2_xlsx.columns.tolist(), SingleComparator.result_header) + self.assertEqual(len(output2_xlsx), 5) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/cpp/include/test_utils.cpp b/debug/accuracy_tools/msprobe/test/cpp/include/test_utils.cpp index e744233b3199c15f5ce77b4690bbaa523b0bad45..08dddbed6b7b0c83691826998a2291bb99a40990 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/include/test_utils.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/include/test_utils.cpp @@ -2,7 +2,6 @@ #include #include #include -#include std::string TEST_ExecShellCommand(const std::string& cmd) { @@ -18,11 +17,10 @@ std::string TEST_ExecShellCommand(const std::string& cmd) return result; } -std::string trim(const std::string& str) +std::string Trim(const std::string& str) { std::string::size_type first = str.find_first_not_of(" \t\n\r\f\v"); std::string::size_type last = str.find_last_not_of(" \t\n\r\f\v"); - if (first == std::string::npos || last == std::string::npos) { return ""; } diff --git a/debug/accuracy_tools/msprobe/test/cpp/include/test_utils.hpp b/debug/accuracy_tools/msprobe/test/cpp/include/test_utils.hpp index ed842b87db77e75e618acd7a25949145a1578c37..08326522a9b06b671e62b5ecacbcc722f485f439 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/include/test_utils.hpp +++ b/debug/accuracy_tools/msprobe/test/cpp/include/test_utils.hpp @@ -5,4 +5,4 @@ #define CONFIG_EXAMPLE __RESOURCES_PATH__"/config.json" std::string TEST_ExecShellCommand(const std::string& cmd); -std::string trim(const std::string& str); +std::string Trim(const std::string& str); diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_config.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_config.cpp index e8b9b73fb66c3fcae40819545c84b7fafb5d2c4d..36033c06a7336ec673eacf24c0c964b6a7719b59 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_config.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_config.cpp @@ -2,14 +2,14 @@ #include "gtest/gtest.h" #include "nlohmann/json.hpp" #include "test_utils.hpp" -#include "base/ErrorInfos.hpp" -#include "base/DebuggerConfig.hpp" +#include "base/ErrorInfosManager.h" +#include "base/DebuggerConfig.h" using namespace MindStudioDebugger; namespace MsProbeTest { -static const std::string cfgContent = R"({ +static const std::string CFG_CONTENT = R"({ "task": "statistics", "dump_path": "./dump_path", "rank": [], @@ -104,7 +104,7 @@ void TestConfigMindSpore::SetUp() DebuggerConfig::GetInstance().Reset(); CleanErrorInfoCache(); ErrorInfosManager::SetLogPath(logpath); - cfgJson = nlohmann::json::parse(cfgContent); + cfgJson = nlohmann::json::parse(CFG_CONTENT); } void TestConfigMindSpore::TearDown() @@ -173,7 +173,7 @@ TEST_F(TestConfigMindSpore, TestCommonCfg) ASSERT_EQ(DumpCfgFile(), 0); EXPECT_EQ(cfg.LoadConfig(framework, cfgPath), 0); EXPECT_EQ(cfg.GetTaskList(), std::vector({DebuggerTaskType::TASK_DUMP_STATISTICS})); - EXPECT_EQ(cfg.GetOutputPath(), trim(TEST_ExecShellCommand("realpath ./output1"))); + EXPECT_EQ(cfg.GetOutputPath(), Trim(TEST_ExecShellCommand("realpath ./output1"))); EXPECT_EQ(cfg.GetRankRange(), std::vector({0, 1, 8})); EXPECT_EQ(cfg.GetStepRange(), std::vector({2, 4, 6, 7, 8})); EXPECT_EQ(cfg.GetDebugLevel(), DebuggerLevel::L2); diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_cpython_utils.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_cpython_utils.cpp index 0d9188878c0864d66d76cc3a823b0a0a5cf644d5..8bb5af7123f41fb42091c2cb21d394bce2b1af8d 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_cpython_utils.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_cpython_utils.cpp @@ -2,7 +2,7 @@ #include #include "test_utils.hpp" -#include "utils/CPythonUtils.hpp" +#include "utils/CPythonUtils.h" using namespace MindStudioDebugger; using namespace MindStudioDebugger::CPythonUtils; @@ -56,79 +56,79 @@ TEST_F(CPythonUtilsTest, CPythonAgent) { TEST_F(CPythonUtilsTest, PythonObjectFromTo) { // 测试PythonObject的From和To函数 - int32_t input_int = -42; - PythonObject obj_int = PythonObject::From(input_int); - EXPECT_TRUE(obj_int.IsNumber()); + int32_t inputInt = -42; + PythonObject objInt = PythonObject::From(inputInt); + EXPECT_TRUE(objInt.IsNumber()); - int32_t output_int; - EXPECT_EQ(obj_int.To(output_int), 0); - EXPECT_EQ(output_int, input_int); + int32_t outputInt; + EXPECT_EQ(objInt.To(outputInt), 0); + EXPECT_EQ(outputInt, inputInt); - uint32_t input_uint = 56; - PythonObject obj_uint = PythonObject::From(input_uint); - EXPECT_TRUE(obj_uint.IsNumber()); + uint32_t inputUint = 56; + PythonObject objUint = PythonObject::From(inputUint); + EXPECT_TRUE(objUint.IsNumber()); - uint32_t output_uint; - EXPECT_EQ(obj_uint.To(output_uint), 0); - EXPECT_EQ(output_uint, input_uint); + uint32_t outputUint; + EXPECT_EQ(objUint.To(outputUint), 0); + EXPECT_EQ(outputUint, inputUint); - double input_double = 3.14; - PythonObject obj_double = PythonObject::From(input_double); - EXPECT_TRUE(obj_double.IsNumber()); + double inputDouble = 3.14; + PythonObject objDouble = PythonObject::From(inputDouble); + EXPECT_TRUE(objDouble.IsNumber()); - double output_double; - EXPECT_EQ(obj_double.To(output_double), 0); - EXPECT_DOUBLE_EQ(output_double, input_double); + double outputDouble; + EXPECT_EQ(objDouble.To(outputDouble), 0); + EXPECT_DOUBLE_EQ(outputDouble, inputDouble); - std::string input_str = "hello"; - PythonObject obj_str = PythonObject::From(input_str); - EXPECT_TRUE(obj_str.IsString()); + std::string inputStr = "hello"; + PythonObject objStr = PythonObject::From(inputStr); + EXPECT_TRUE(objStr.IsString()); - std::string output_str; - EXPECT_EQ(obj_str.To(output_str), 0); - EXPECT_EQ(output_str, input_str); + std::string outputStr; + EXPECT_EQ(objStr.To(outputStr), 0); + EXPECT_EQ(outputStr, inputStr); - const char* input_char = "world"; - PythonObject obj_str1 = PythonObject::From(input_char); - EXPECT_TRUE(obj_str1.IsString()); + const char* inputChar = "world"; + PythonObject objStr1 = PythonObject::From(inputChar); + EXPECT_TRUE(objStr1.IsString()); - EXPECT_EQ(obj_str1.To(output_str), 0); - EXPECT_EQ(output_str, std::string(input_char)); + EXPECT_EQ(objStr1.To(outputStr), 0); + EXPECT_EQ(outputStr, std::string(inputChar)); - bool input_bool = true; - PythonObject obj_bool = PythonObject::From(input_bool); - EXPECT_TRUE(obj_bool.IsBool()); + bool inputBool = true; + PythonObject objBool = PythonObject::From(inputBool); + EXPECT_TRUE(objBool.IsBool()); - bool output_bool; - EXPECT_EQ(obj_bool.To(output_bool), 0); - EXPECT_EQ(output_bool, input_bool); + bool outputBool; + EXPECT_EQ(objBool.To(outputBool), 0); + EXPECT_EQ(outputBool, inputBool); - std::vector input_vector_int = {1, 2, 3, 100}; - PythonObject list_int_obj = PythonObject::From(input_vector_int); - EXPECT_TRUE(list_int_obj.IsList()); + std::vector inputVectorInt = {1, 2, 3, 100}; + PythonObject listIntObj = PythonObject::From(inputVectorInt); + EXPECT_TRUE(listIntObj.IsList()); - std::vector output_vector_int; - EXPECT_EQ(list_int_obj.To(output_vector_int), 0); + std::vector outputVectorInt; + EXPECT_EQ(listIntObj.To(outputVectorInt), 0); - size_t size = input_vector_int.size(); - EXPECT_EQ(size, output_vector_int.size()); + size_t size = inputVectorInt.size(); + EXPECT_EQ(size, outputVectorInt.size()); for (size_t i = 0; i < size; ++i) { - EXPECT_EQ(input_vector_int[i], output_vector_int[i]); + EXPECT_EQ(inputVectorInt[i], outputVectorInt[i]); } - std::vector input_vector_str = {"a", "bb", "ccc", "dddd"}; - PythonObject list_str_obj = PythonObject::From(input_vector_str); - EXPECT_TRUE(list_str_obj.IsList()); + std::vector inputVectorStr = {"a", "bb", "ccc", "dddd"}; + PythonObject listStrObj = PythonObject::From(inputVectorStr); + EXPECT_TRUE(listStrObj.IsList()); - std::vector output_vector_str; - EXPECT_EQ(list_str_obj.To(output_vector_str), 0); + std::vector outputVectorStr; + EXPECT_EQ(listStrObj.To(outputVectorStr), 0); - size = input_vector_str.size(); - EXPECT_EQ(size, output_vector_str.size()); + size = inputVectorStr.size(); + EXPECT_EQ(size, outputVectorStr.size()); for (size_t i = 0; i < size; ++i) { - EXPECT_EQ(input_vector_str[i], output_vector_str[i]); + EXPECT_EQ(inputVectorStr[i], outputVectorStr[i]); } } @@ -199,18 +199,18 @@ TEST_F(CPythonUtilsTest, PythonNumberObject) { PythonNumberObject o5(PythonObject::From(4.44)); PythonNumberObject o6(PythonObject::From("1111")); - int int_v; - EXPECT_EQ(o1.To(int_v), 0); - EXPECT_EQ(int_v, 123); - double double_v; - EXPECT_EQ(o2.To(double_v), 0); - EXPECT_TRUE(std::fabs(double_v - 3.14) < 1e-5); - EXPECT_EQ(o3.To(int_v), 0); - EXPECT_EQ(int_v, 321); - EXPECT_EQ(o4.To(double_v), 0); - EXPECT_TRUE(std::fabs(double_v - 2.33) < 1e-5); - EXPECT_EQ(o5.To(double_v), 0); - EXPECT_TRUE(std::fabs(double_v - 4.44) < 1e-5); + int intV; + EXPECT_EQ(o1.To(intV), 0); + EXPECT_EQ(intV, 123); + double doubleV; + EXPECT_EQ(o2.To(doubleV), 0); + EXPECT_TRUE(std::fabs(doubleV - 3.14) < 1e-5); + EXPECT_EQ(o3.To(intV), 0); + EXPECT_EQ(intV, 321); + EXPECT_EQ(o4.To(doubleV), 0); + EXPECT_TRUE(std::fabs(doubleV - 2.33) < 1e-5); + EXPECT_EQ(o5.To(doubleV), 0); + EXPECT_TRUE(std::fabs(doubleV - 4.44) < 1e-5); EXPECT_TRUE(o6.IsNone()); } diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_data_utils.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_data_utils.cpp index 11442f12bfea9179ecd4e2e357bcf70b4212ab84..dd6325c2183332bfa0cc2acc592301f7cf58bda8 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_data_utils.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_data_utils.cpp @@ -2,7 +2,7 @@ #include #include #include -#include "utils/DataUtils.hpp" +#include "utils/DataUtils.h" using namespace MindStudioDebugger; using namespace MindStudioDebugger::DataUtils; @@ -10,15 +10,15 @@ using namespace MindStudioDebugger::DataUtils; namespace MsProbeTest { TEST(DataUtilsTest, TestUnpackUint64Value) { - uint64_t data_le = 0x0102030405060708; - uint64_t result = UnpackUint64Value_Le(&data_le); + uint64_t dataLe = 0x0102030405060708; + uint64_t result = UnpackUint64ValueLe(&dataLe); #if __BYTE_ORDER == __LITTLE_ENDIAN EXPECT_EQ(result, 0x0102030405060708); #else EXPECT_EQ(result, 0x0807060504030201); #endif - uint64_t data_be = 0x0102030405060708; - result = UnpackUint64Value_Be(&data_be); + uint64_t dataBe = 0x0102030405060708; + result = UnpackUint64ValueBe(&dataBe); #if __BYTE_ORDER == __LITTLE_ENDIAN EXPECT_EQ(result, 0x0807060504030201); #else @@ -74,7 +74,7 @@ TEST(DataUtilsTest, TestGetFormatString) { EXPECT_EQ(GetFormatString(TensorFormat::FORMAT_FRACTAL_Z), "FRACTAL_Z"); EXPECT_EQ(GetFormatString(TensorFormat::FORMAT_C1HWNC0), "C1HWNC0"); EXPECT_EQ(GetFormatString(TensorFormat::FORMAT_HWCN), "HWCN"); - EXPECT_EQ(GetFormatString(TensorFormat::FORMAT_C1HWNCoC0), "C1HWNCoC0"); + EXPECT_EQ(GetFormatString(TensorFormat::FORMAT_C1HWNCOC0), "C1HWNCoC0"); EXPECT_EQ(GetFormatString(TensorFormat::FORMAT_DHWNC), "DHWNC"); EXPECT_EQ(GetFormatString(TensorFormat::FORMAT_NCL), "NCL"); EXPECT_EQ(GetFormatString(TensorFormat::FORMAT_MAX), "UNKNOWN"); diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_environ.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_environ.cpp index 94c830227ae58637642a189f36ade78de9a2a75c..be30c5c219ce1b34b92a1453eeb2050479bc7b97 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_environ.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_environ.cpp @@ -2,8 +2,8 @@ #include #include "include/test_utils.hpp" -#include "base/DebuggerConfig.hpp" -#include "base/Environment.hpp" +#include "base/DebuggerConfig.h" +#include "base/Environment.h" using namespace MindStudioDebugger; using namespace MindStudioDebugger::Environment; diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_file_operation.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_file_operation.cpp index 2886126e9f568fba6b8ce3eabd752653d4493108..99dbe8124d17fe7cdcfd1f812d2132f416cea1ea 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_file_operation.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_file_operation.cpp @@ -4,8 +4,8 @@ #include #include "test_utils.hpp" -#include "utils/DataUtils.hpp" -#include "utils/FileOperation.hpp" +#include "utils/DataUtils.h" +#include "utils/FileOperation.h" using namespace MindStudioDebugger; using namespace MindStudioDebugger::FileOperation; diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_file_utils.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_file_utils.cpp index 03449f761be0c8548021218581f4cbff12d4e07d..022ae396ba3d7a8343792a438162a74ff526fc75 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_file_utils.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_file_utils.cpp @@ -8,7 +8,7 @@ #include #include "test_utils.hpp" -#include "utils/FileUtils.hpp" +#include "utils/FileUtils.h" using namespace MindStudioDebugger; using namespace MindStudioDebugger::FileUtils; @@ -52,7 +52,7 @@ TEST_F(FileUtilsTest, TestIsPathExist) TEST_F(FileUtilsTest, TestGetAbsPath) { - std::string pwd = trim(TEST_ExecShellCommand("pwd")); + std::string pwd = Trim(TEST_ExecShellCommand("pwd")); EXPECT_EQ(pwd, GetAbsPath(".")); EXPECT_EQ(pwd + "/testpath", GetAbsPath("./testpath")); EXPECT_EQ(pwd + "/testpath", GetAbsPath("./testpath/")); @@ -210,8 +210,8 @@ TEST_F(FileUtilsTest, TestIsPathLengthLegal) TEST_F(FileUtilsTest, TestIsPathDepthValid) { EXPECT_TRUE(IsPathDepthValid("")); - EXPECT_TRUE(IsPathDepthValid(std::string(PATH_DEPTH_MAX, pathSeparator))); - EXPECT_FALSE(IsPathDepthValid(std::string(PATH_DEPTH_MAX + 1, pathSeparator))); + EXPECT_TRUE(IsPathDepthValid(std::string(PATH_DEPTH_MAX, PATH_SEPARATOR))); + EXPECT_FALSE(IsPathDepthValid(std::string(PATH_DEPTH_MAX + 1, PATH_SEPARATOR))); } TEST_F(FileUtilsTest, TestIsFileOwner) diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_log.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_log.cpp index 254b54359a50166e1d893c5b936eb220ee0b2a73..ddf6950fd5b6ff0c7f191e5aa0f8e79897db6c7c 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_log.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_log.cpp @@ -2,7 +2,7 @@ #include "gtest/gtest.h" #include "test_utils.hpp" -#include "base/ErrorInfos.hpp" +#include "base/ErrorInfosManager.h" using namespace MindStudioDebugger; diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_math_utils.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_math_utils.cpp index 3b23e9c879c431ef7457990ba774aa0dc1321b45..8e57d2cd53b5c52ca62084e8ecbd49e3e8138682 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_math_utils.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_math_utils.cpp @@ -3,7 +3,7 @@ #include #include #include -#include "utils/MathUtils.hpp" +#include "utils/MathUtils.h" using namespace MindStudioDebugger; using namespace MindStudioDebugger::MathUtils; diff --git a/debug/accuracy_tools/msprobe/test/cpp/test_precision_debugger.cpp b/debug/accuracy_tools/msprobe/test/cpp/test_precision_debugger.cpp index 69df0c18fcc27cd0ac359262649fcc588f2e9b9f..2832f2345d7d72efe2d1bb305c044f56d9b83e8d 100644 --- a/debug/accuracy_tools/msprobe/test/cpp/test_precision_debugger.cpp +++ b/debug/accuracy_tools/msprobe/test/cpp/test_precision_debugger.cpp @@ -2,9 +2,9 @@ #include #include "include/test_utils.hpp" -#include "third_party/ACL/AclApi.hpp" -#include "base/ErrorInfos.hpp" -#include "core/PrecisionDebugger.hpp" +#include "third_party/ACL/AclApi.h" +#include "base/ErrorInfosManager.h" +#include "core/PrecisionDebugger.h" using namespace MindStudioDebugger; @@ -17,15 +17,15 @@ public: std::string Name() const override {return "PrecisionDbgTaskStub";} bool Condition(const DebuggerConfig& cfg) const override {return true;} - void Initialize(const DebuggerConfig& cfg) {initialize_called = true;} - void OnStart() {start_called = true;} - void OnStop() {stop_called = true;} - void OnStep() {step_called = true;} + void Initialize(const DebuggerConfig& cfg) {initializeCalled = true;} + void OnStart() {startCalled = true;} + void OnStop() {stopCalled = true;} + void OnStep() {stepCalled = true;} - bool initialize_called{false}; - bool start_called{false}; - bool stop_called{false}; - bool step_called{false}; + bool initializeCalled{false}; + bool startCalled{false}; + bool stopCalled{false}; + bool stepCalled{false}; }; class PrecisionDbgTaskUselessStub : public PrecisionDbgTaskStub { @@ -35,11 +35,11 @@ public: TEST(PrecisionDebuggerTest, TestRegisterBeforeInit) { PrecisionDebugger& debugger = PrecisionDebugger::GetInstance(); - PrecisionDbgTaskStub stub_task; + PrecisionDbgTaskStub stubTask; DebuggerConfig::GetInstance().Reset(); - debugger.RegisterDebuggerTask(&stub_task); - stub_task.Register(); + debugger.RegisterDebuggerTask(&stubTask); + stubTask.Register(); EXPECT_FALSE(debugger.IsEnable()); EXPECT_EQ(debugger.GetCurStep(), 0); @@ -49,12 +49,12 @@ TEST(PrecisionDebuggerTest, TestRegisterBeforeInit) { debugger.Step(); EXPECT_EQ(debugger.GetCurStep(), 0); - EXPECT_FALSE(stub_task.initialize_called); - EXPECT_FALSE(stub_task.start_called); - EXPECT_FALSE(stub_task.stop_called); - EXPECT_FALSE(stub_task.step_called); + EXPECT_FALSE(stubTask.initializeCalled); + EXPECT_FALSE(stubTask.startCalled); + EXPECT_FALSE(stubTask.stopCalled); + EXPECT_FALSE(stubTask.stepCalled); - debugger.UnRegisterDebuggerTask(&stub_task); + debugger.UnRegisterDebuggerTask(&stubTask); debugger.UnRegisterDebuggerTask(nullptr); } @@ -81,39 +81,39 @@ TEST(PrecisionDebuggerTest, TestInit) { TEST(PrecisionDebuggerTest, TestSubTaskDispatch) { PrecisionDebugger& debugger = PrecisionDebugger::GetInstance(); - PrecisionDbgTaskStub stub_task1; - PrecisionDbgTaskStub stub_task2; - PrecisionDbgTaskUselessStub stub_task3; + PrecisionDbgTaskStub stubTask1; + PrecisionDbgTaskStub stubTask2; + PrecisionDbgTaskUselessStub stubTask3; MOCKER(MindStudioDebugger::AscendCLApi::LoadAclApi) .stubs() .then(returnValue(0)); - MOCKER(MindStudioDebugger::AscendCLApi::ACLAPI_aclrtSynchronizeDevice) + MOCKER(MindStudioDebugger::AscendCLApi::AclApiAclrtSynchronizeDevice) .stubs() .then(returnValue(0)) .expects(atLeast(1)); - stub_task1.Register(); + stubTask1.Register(); EXPECT_EQ(debugger.Initialize("MindSpore", CONFIG_EXAMPLE), 0); - stub_task2.Register(); - stub_task3.Register(); + stubTask2.Register(); + stubTask3.Register(); - EXPECT_TRUE(stub_task1.initialize_called); - EXPECT_TRUE(stub_task2.initialize_called); - EXPECT_FALSE(stub_task3.initialize_called); - EXPECT_FALSE(stub_task1.start_called); - EXPECT_FALSE(stub_task2.stop_called); - EXPECT_FALSE(stub_task3.step_called); + EXPECT_TRUE(stubTask1.initializeCalled); + EXPECT_TRUE(stubTask2.initializeCalled); + EXPECT_FALSE(stubTask3.initializeCalled); + EXPECT_FALSE(stubTask1.startCalled); + EXPECT_FALSE(stubTask2.stopCalled); + EXPECT_FALSE(stubTask3.stepCalled); debugger.Start(); - EXPECT_TRUE(stub_task1.start_called); - EXPECT_FALSE(stub_task3.start_called); + EXPECT_TRUE(stubTask1.startCalled); + EXPECT_FALSE(stubTask3.startCalled); debugger.Stop(); - EXPECT_TRUE(stub_task1.stop_called); - EXPECT_TRUE(stub_task2.stop_called); + EXPECT_TRUE(stubTask1.stopCalled); + EXPECT_TRUE(stubTask2.stopCalled); debugger.Step(); - EXPECT_TRUE(stub_task1.step_called); + EXPECT_TRUE(stubTask1.stepCalled); GlobalMockObject::verify(); GlobalMockObject::reset(); diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py index 2cf47a2064626db792c55efb449b47ca8ab9b04e..7214dfab3885689b4c5b0c903e430ec9dcecb989 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py @@ -4,6 +4,9 @@ import logging import os import json import csv +import tempfile +import shutil + from msprobe.mindspore.api_accuracy_checker.api_accuracy_checker import ApiAccuracyChecker @@ -40,13 +43,42 @@ def find_with_prefix(directory, prefix): class Args: - def __init__(self, api_info_file=None, out_path=None, result_csv_path=None): + def __init__(self, api_info_file=None, out_path=None, result_csv_path=None, save_error_data=False): self.api_info_file = api_info_file if api_info_file is not None else os.path.join(directory, "files", "api_info_statistics.json") self.out_path = out_path if out_path is not None else os.path.join(directory, "files") self.result_csv_path = result_csv_path if result_csv_path is not None else "" + self.save_error_data = save_error_data class TestApiAccuracyChecker(unittest.TestCase): + def test_init_save_error_data(self): + # 使用临时目录，不污染项目文件 + temp_dir = tempfile.mkdtemp() + try: + # 构造 args，只关注 out_path 和 save_error_data + args = Args(out_path=temp_dir, save_error_data=True) + config, dump_path_agg = ApiAccuracyChecker.init_save_error_data(args) + + # 1. config 字段检查 + self.assertEqual(config.execution_mode, "pynative") + self.assertEqual(config.task, "tensor") + self.assertEqual(config.dump_path, temp_dir) + self.assertEqual(config.dump_tensor_data_dir, temp_dir) + self.assertFalse(config.async_dump) + self.assertEqual(config.file_format, "npy") + + # 2. error_data 目录已创建 + error_dir = os.path.join(temp_dir, "error_data") + self.assertTrue(os.path.isdir(error_dir), f"{error_dir} should exist") + + # 3. dump_path_agg 路径检查 + self.assertEqual(dump_path_agg.dump_file_path, os.path.join(temp_dir, "dump.json")) + self.assertEqual(dump_path_agg.stack_file_path, os.path.join(temp_dir, "stack.json")) + self.assertEqual(dump_path_agg.dump_tensor_data_dir, error_dir) + + finally: + # 清理临时目录 + shutil.rmtree(temp_dir) def test_statistics_mode(self): api_info_statistics_path = os.path.join(directory, "files", "api_info_statistics.json") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py index dac2b9b364f6e4f271ff5ae8a43052a3fd2496d2..8d6387bb73bcf455b4066ee6796ac3227b15f10d 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py @@ -121,10 +121,20 @@ class TestApiRunner(unittest.TestCase): ] for test_case in test_cases: api_instance, api_input_aggregation, forward_or_backward, api_platform, results_target = test_case - results_real = api_runner.run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform) + output = api_runner.run_api(api_instance, api_input_aggregation, forward_or_backward, + api_platform) + + # 如果返回的是 tuple，就拿第一个元素；否则直接当 list 用 + if isinstance(output, tuple): + results_real = output[0] + else: + results_real = output + # 下面跟原来测试逻辑一模一样 for res_real, res_target in zip(results_real, results_target): - assert (abs(res_real.get_parameter() - res_target.get_parameter(tensor_platform=api_platform)) < 1e-5).all() - + assert (abs( + res_real.get_parameter() + - res_target.get_parameter(tensor_platform=api_platform) + ) < 1e-5).all() def test_get_api_instance(self): #api_type_str, api_sub_name, api_platform, result_api diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_op_generate.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_op_generate.py new file mode 100644 index 0000000000000000000000000000000000000000..9166074ce6dfd0f6a794d5fd1b196495450541f7 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_op_generate.py @@ -0,0 +1,309 @@ +import unittest +import tempfile +import os +import json + +from msprobe.core.common.const import Const +from msprobe.mindspore.api_accuracy_checker.generate_op_script.op_generator import ( + APIInfo, + CommonConfig, + parse_json_config, + OperatorScriptGenerator, + APIExtractor, +) +from msprobe.core.common.file_utils import ( + FileOpen, + load_json, + save_json, + make_dir, + change_mode, +) +from msprobe.core.common.const import FileCheckConst + +class TestCommonConfigCheckConfig(unittest.TestCase): + def setUp(self): + # 基本有效配置 + self.tmpdir = tempfile.TemporaryDirectory() + self.valid = { + "dump_json_path": None, + "api_name": "Functional.add", + "extract_api_path": os.path.join(self.tmpdir.name, "out.json"), + "propagation": Const.FORWARD, + "data_mode": "random_data", + "random_seed": 0, + "iter_times": 1, + } + + def tearDown(self): + self.tmpdir.cleanup() + + def make_cfg(self, overrides): + cfg_dict = {**self.valid, **overrides} + return CommonConfig(cfg_dict) + + def test_invalid_api_name_too_long(self): + long_name = "A" * 31 + with self.assertRaises(ValueError) as cm: + self.make_cfg({"api_name": long_name}) + self.assertIn("too long", str(cm.exception)) + + def test_invalid_propagation(self): + with self.assertRaises(ValueError): + self.make_cfg({"propagation": "INVALID"}) + + def test_invalid_data_mode(self): + with self.assertRaises(ValueError): + self.make_cfg({"data_mode": "not_a_mode"}) + + def test_random_seed_not_int(self): + with self.assertRaises(ValueError): + self.make_cfg({"random_seed": "zero"}) + + def test_iter_times_not_int(self): + with self.assertRaises(ValueError): + self.make_cfg({"iter_times": "ten"}) + + +class TestParseJsonConfig(unittest.TestCase): + def test_empty_path_raises(self): + with self.assertRaises(Exception) as cm: + parse_json_config("") # 空路径 + self.assertIn("config_input path can not be empty", str(cm.exception)) + +class TestAPIExtractorExtractOp(unittest.TestCase): + def setUp(self): + # 准备一个 dump_json_path 文件 + self.tmpdir = tempfile.TemporaryDirectory() + self.dump = { + "framework": "mindspore", + "dump_data_dir": "/data", + "data": { + "Functional.add.0": { + Const.INPUT_ARGS: [{"data_name": "a.bin"}], + Const.OUTPUT: [{"data_name": "b.bin"}] + }, + "Other.mul.1": { + Const.INPUT_ARGS: [{"data_name": "c.bin"}] + } + } + } + self.dump_path = os.path.join(self.tmpdir.name, "dump.json") + with open(self.dump_path, "w") as f: + json.dump(self.dump, f) + # 输出路径 + self.out_path = os.path.join(self.tmpdir.name, "extract.json") + + def tearDown(self): + self.tmpdir.cleanup() + + def test_extract_op_creates_file_with_expected_keys(self): + extractor = APIExtractor("Functional.add", self.dump_path, self.out_path) + extractor.extract_op() + # 文件存在 + self.assertTrue(os.path.isfile(self.out_path)) + data = load_json(self.out_path) + # 应包含匹配 key 以及 FRAMEWORK、REAL_DATA_PATH + self.assertIn("Functional.add.0", data) + self.assertEqual(data.get("framework"), "mindspore") + self.assertEqual(data.get("real_data_path"), "/data") + # data_name 已被拼接 + arg = data["Functional.add.0"][Const.INPUT_ARGS][0] + self.assertEqual(arg["data_name"], os.path.join("/data", "a.bin")) + +class TestAPIExtractorUpdateDataNameNested(unittest.TestCase): + def test_update_data_name_nested_list(self): + ex = APIExtractor("Any", None, None) + data = {"data_name": "root"} + nested = [ [data], [{"data_name": "leaf"}] ] + ex.update_data_name(nested, "/base") + # 所有层级的 data_name 都被更新 + self.assertEqual(nested[0][0]["data_name"], "/base/root") + self.assertEqual(nested[1][0]["data_name"], "/base/leaf") + +class TestOperatorScriptGeneratorSegments(unittest.TestCase): + def test_extract_segments_invalid_length(self): + # 既不是 4 段也不是 5 段 + t, name, order = OperatorScriptGenerator.extract_detailed_api_segments("a.b.c") + self.assertIsNone(t) + self.assertIsNone(name) + self.assertIsNone(order) + +class TestOperatorScriptGeneratorNestedInputs(unittest.TestCase): + def test_generate_forward_inputs_code_nested(self): + args = [ + {"parameter_name": "x"}, + [ {"parameter_name": "y1"}, {"parameter_name": "y2"} ], + ] + code = OperatorScriptGenerator.generate_forward_inputs_code(args) + self.assertIn("x", code) + self.assertIn("y1", code) + self.assertIn("y2", code) + + def test_generate_gradient_inputs_code_nested(self): + args = [ + {"parameter_name": "g1"}, + [ {"parameter_name": "g2"} ] + ] + code = OperatorScriptGenerator.generate_gradient_inputs_code(args) + self.assertIn("g1", code) + self.assertIn("g2", code) + + +class TestAPIInfo(unittest.TestCase): + def test_api_type_and_supported(self): + api = APIInfo("Functional.add.0.forward", {}) + self.assertEqual(api.api_type, "Functional") + self.assertTrue(api.is_supported_type()) + + def test_from_json_forward(self): + data = {"Functional.add.0": {"input_args": [], "input_kwargs": {}}} + info = APIInfo.from_json(data, Const.FORWARD) + self.assertEqual(info.api_full_name, "Functional.add.0") + self.assertIsNone(info.backward_info) + + def test_from_json_backward(self): + data = { + "Functional.add.0": {"input_args": [], "input_kwargs": {}}, + "Functional.add_grad.0": {"grad_input": []}, + } + info = APIInfo.from_json(data, Const.BACKWARD) + self.assertEqual(info.api_full_name, "Functional.add.0") + self.assertIsNotNone(info.backward_info) + self.assertEqual(info.backward_info.api_full_name, "Functional.add_grad.0") + + def test_from_json_unsupported_type(self): + data = {"Unknown.add.0": {}} + with self.assertRaises(ValueError): + APIInfo.from_json(data, Const.FORWARD) + + +class TestCommonConfig(unittest.TestCase): + def setUp(self): + # create a temp directory to satisfy make_dir and path checks + self.tmpdir = tempfile.TemporaryDirectory() + self.extract_path = os.path.join(self.tmpdir.name, "sub", "api.json") + # build a valid config dict + self.config = { + "dump_json_path": None, + "api_name": "Functional.add", + "extract_api_path": self.extract_path, + "propagation": Const.FORWARD, + "data_mode": "random_data", + "random_seed": 1, + "iter_times": 1, + } + # ensure parent dir of extract_api_path exists + os.makedirs(os.path.dirname(self.extract_path), exist_ok=True) + # write a dummy JSON file for parse_json_config + self.config_file = os.path.join(self.tmpdir.name, "config.json") + with open(self.config_file, "w") as f: + json.dump(self.config, f) + + def tearDown(self): + self.tmpdir.cleanup() + + def test_parse_json_config(self): + cfg = parse_json_config(self.config_file) + self.assertIsInstance(cfg, CommonConfig) + self.assertEqual(cfg.api_name, "Functional.add") + self.assertEqual(cfg.propagation, Const.FORWARD) + + def test_check_user_settings_invalid_iter(self): + cfg = CommonConfig(self.config.copy()) + cfg.iter_times = 0 + with self.assertRaises(ValueError) as ctx: + cfg.check_user_settings() + self.assertIn("iter_times should be range from 1", str(ctx.exception)) + + def test_check_user_settings_empty_json(self): + # create an empty JSON file to simulate empty extract_api_path + empty = os.path.join(self.tmpdir.name, "empty.json") + with open(empty, "w") as f: + json.dump({}, f) + cfg = CommonConfig({**self.config, "extract_api_path": empty}) + with self.assertRaises(ValueError) as ctx: + cfg.check_user_settings() + self.assertIn("json file is empty", str(ctx.exception)) + + +class TestOperatorScriptGenerator(unittest.TestCase): + def test_extract_detailed_api_segments_four(self): + t, name, order = OperatorScriptGenerator.extract_detailed_api_segments( + "Functional.mul.1.out" + ) + self.assertEqual((t, name, order), ("Functional", "mul", "1")) + + def test_extract_detailed_api_segments_five(self): + t, name, order = OperatorScriptGenerator.extract_detailed_api_segments( + "Functional.prefix.mul.2.out" + ) + self.assertEqual((t, name, order), ("Functional", "prefix.mul", "2")) + + def test_generate_forward_inputs_code(self): + args_info = [{"parameter_name": "x"}, {"parameter_name": "y"}] + code = OperatorScriptGenerator.generate_forward_inputs_code(args_info) + self.assertIn("x", code) + self.assertIn("y", code) + self.assertIn("ComputeElement", code) + + def test_generate_kwargs_compute_element_dict_code(self): + code = OperatorScriptGenerator.generate_kwargs_compute_element_dict_code() + self.assertIn("kwargs_compute_element_dict", code) + self.assertTrue(code.strip().startswith("# ---- 构造 kwargs")) + + def test_generate_gradient_inputs_code(self): + args_back = [{"parameter_name": "grad"}] + code = OperatorScriptGenerator.generate_gradient_inputs_code(args_back) + self.assertIn("grad", code) + self.assertIn("gradient_inputs", code) + + def test_get_settings_real_data(self): + # simulate CommonConfig-like object + common = type("C", (), { + "propagation": Const.FORWARD, + "random_seed": 42, + "data_mode": "real_data", + "iter_times": 100 + }) + gen = OperatorScriptGenerator(common, ["a"], {"k": "v"}, None) + settings = gen.get_settings("Functional.add.0") + self.assertEqual(settings["iter_times"], 1) + self.assertEqual(settings["random_seed"], 42) + + def test_get_settings_random_data(self): + common = type("C", (), { + "propagation": Const.FORWARD, + "random_seed": 7, + "data_mode": "random_data", + "iter_times": 5 + }) + gen = OperatorScriptGenerator(common, ["a"], {"k": "v"}, None) + settings = gen.get_settings("Tensor.sub.3") + self.assertEqual(settings["iter_times"], 5) + + +class TestAPIExtractor(unittest.TestCase): + def test_update_data_name_simple(self): + ex = APIExtractor("Functional.add", None, None) + data = {"data_name": "foo.bin"} + ex.update_data_name(data, "/dumpdir") + self.assertEqual(data["data_name"], os.path.join("/dumpdir", "foo.bin")) + + def test_load_real_data_path(self): + ex = APIExtractor("Functional.add", None, None) + # build a minimal value dict + val = { + Const.INPUT_ARGS: [{"data_name": "a.txt"}], + Const.GRAD_INPUT: [], + Const.INPUT: [], + Const.OUTPUT: [], + Const.GRAD_OUTPUT: [] + } + out = ex.load_real_data_path(val, "/mydump") + # ensure in-place mutation happened + self.assertEqual(val[Const.INPUT_ARGS][0]["data_name"], "/mydump/a.txt") + self.assertIs(out, val) + + +if __name__ == "__main__": + unittest.main() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py index 6f7377894002e60add41dc7b2d3c1d3d68391e0b..eafe9384618502390b41adedd7d32db172ca8188 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py @@ -1,21 +1,18 @@ # coding=utf-8 -import json import os -import random import shutil -import tempfile +import random import unittest from unittest.mock import patch -import numpy as np -import pandas as pd import torch -import yaml +import numpy as np -from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import ModeConfig -from msprobe.mindspore.compare.ms_compare import MappingConfig, MSComparator, check_cross_framework +from msprobe.mindspore.compare.ms_compare import check_cross_framework, read_real_data, ms_compare from msprobe.core.common.const import Const +from msprobe.test.core_ut.compare.test_acc_compare import generate_dump_json, generate_stack_json +from msprobe.core.common.utils import CompareException + npu_dict = {'op_name': ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.1', 'Functional.conv2d.0.forward.input.2', 'Functional.conv2d.0.forward.output'], @@ -175,6 +172,8 @@ json_data_template = { 'data': {} } +base_dir1 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_ms_compare1') + def gen_data(is_ms=True): type_value = 'mindspore.Tensor' if is_ms else 'torch.Tensor' @@ -190,169 +189,18 @@ def gen_data(is_ms=True): } -def gen_api_mapping_test_data(need_user_mapping=False): - result_npu = json_data_template.copy() - result_bench = json_data_template.copy() - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - ms_comparator = MSComparator(mode_config, mapping_config) - - api_mapping = ms_comparator.load_internal_api() - ms_api_list = np.random.choice(list(api_mapping.keys()), size=5, replace=False).astype(str).tolist() - ms_api_data = {} - pt_api_data = {} - user_mapping = [] - for api in ms_api_list: - call_num = random.randint(1, 10) - direction = random.choice(['forward', 'backward']) - data_name_ms = api + '.' + str(call_num) + '.' + direction - data_name_pt = api_mapping.get(api) + '.' + str(call_num) + '.' + direction - input_num = random.randint(1, 5) - output_num = random.randint(1, 5) - ms_data = {'input_args': [gen_data(True) for _ in range(input_num)], - 'output': [gen_data(True) for _ in range(output_num)]} - pt_data = {'input_args': [gen_data(False) for _ in range(input_num)], - 'output': [gen_data(False) for _ in range(output_num)]} - ms_api_data[data_name_ms] = ms_data - pt_api_data[data_name_pt] = pt_data - if need_user_mapping: - compare_num_input = random.randint(1, input_num) - compare_num_output = random.randint(1, output_num) - user_mapping_item = {'ms_api': api, - 'pt_api': api_mapping.get(api), - 'ms_args': sorted(np.random.choice(list(range(input_num)), size=compare_num_input, - replace=False).astype(int).tolist()), - 'pt_args': sorted(np.random.choice(list(range(input_num)), size=compare_num_input, - replace=False).astype(int).tolist()), - 'ms_output': sorted(np.random.choice(list(range(output_num)), size=compare_num_output, - replace=False).astype(int).tolist()), - 'pt_output': sorted(np.random.choice(list(range(output_num)), size=compare_num_output, - replace=False).astype(int).tolist())} - user_mapping.append(user_mapping_item) - ms_api_key_list = list(ms_api_data.keys()) - random.shuffle(ms_api_key_list) - result_npu['data'] = {k: ms_api_data.get(k) for k in ms_api_key_list} - pt_api_key_list = list(pt_api_data.keys()) - random.shuffle(pt_api_key_list) - result_bench['data'] = {k: pt_api_data.get(k) for k in pt_api_key_list} - return result_npu, result_bench, user_mapping - - class TestUtilsMethods(unittest.TestCase): - def test_check_op_ms(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() + def setUp(self): + os.makedirs(base_dir1, mode=0o750, exist_ok=True) + np.save(os.path.join(base_dir1, 'numpy_data.npy'), np.array([1, 2, 3])) + torch.save(torch.tensor([2, 3, 4]), os.path.join(base_dir1, 'torch_data.pt')) - ms_comparator = MSComparator(mode_config, mapping_config) - result = ms_comparator.check_op(npu_dict, bench_dict) - self.assertTrue(result) + def tearDown(self): + if os.path.exists(base_dir1): + shutil.rmtree(base_dir1) - def test_data_mapping(self): - stack_json_data = {} - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(data_mapping=data_mapping) - ms_comparator = MSComparator(mode_config, mapping_config) - - npu_ops_all = ms_comparator.merge_data(npu_json_data, stack_json_data) - npu_ops_all_correct = { - 'Functional.flash_attention_score.4.forward.input.0': { - 'struct': ('BFloat16', [4096, 1, 2048]), - 'summary': [4.1875, -4.4375, -4.550282028503716e-05, 2316.379150390625], - 'data_name': None, - 'stack_info': [None] - }, - 'Functional.flash_attention_score.4.forward.output.0': { - 'struct': ('BFloat16', [4096, 1, 2048]), - 'summary': [4.1875, -4.4375, -4.550282028503716e-05, 2316.379150390625], - 'data_name': None, - 'stack_info': [None] - } - } - self.assertDictEqual(npu_ops_all, npu_ops_all_correct) - - bench_ops_all = ms_comparator.merge_data(bench_json_data, stack_json_data) - bench_ops_all_correct = { - 'NPU.npu_fusion_attention.4.forward.input.0': { - 'struct': ('torch.bfloat16', [4096, 1, 2048]), - 'summary': [4.1875, -4.4375, -4.553794860839844e-05, 2320.0], - 'data_name': None, - 'stack_info': [None] - }, - 'NPU.npu_fusion_attention.4.forward.output.0': { - 'struct': ('torch.bfloat16', [4096, 1, 2048]), - 'summary': [4.1875, -4.4375, -4.553794860839844e-05, 2320.0], - 'data_name': None, - 'stack_info': [None] - } - } - self.assertDictEqual(bench_ops_all, bench_ops_all_correct) - - result = ms_comparator.get_accuracy(npu_ops_all, bench_ops_all) - result_correct = [['Functional.flash_attention_score.4.forward.input.0', - 'NPU.npu_fusion_attention.4.forward.input.0', - 'BFloat16', 'torch.bfloat16', [4096, 1, 2048], [4096, 1, 2048], 0.0, 0.0, - 3.512832336127758e-08, -3.620849609375, '0.0%', '0.0%', '0.07714076816099476%', - '0.1560711038523707%', 4.1875, -4.4375, -4.550282028503716e-05, 2316.379150390625, - 4.1875, -4.4375, -4.553794860839844e-05, 2320.0, '', '', None], - ['Functional.flash_attention_score.4.forward.output.0', - 'NPU.npu_fusion_attention.4.forward.output.0', - 'BFloat16', 'torch.bfloat16', [4096, 1, 2048], [4096, 1, 2048], 0.0, 0.0, - 3.512832336127758e-08, -3.620849609375, '0.0%', '0.0%', '0.07714076816099476%', - '0.1560711038523707%', 4.1875, -4.4375, -4.550282028503716e-05, 2316.379150390625, - 4.1875, -4.4375, -4.553794860839844e-05, 2320.0, '', '', None] - ] - self.assertListEqual(result, result_correct) - - def test_dm_tensor_task(self): - self.compare_process_custom(dump_mode=Const.ALL) - - def compare_process_custom(self, dump_mode): - data_path = tempfile.mkdtemp(prefix='dump_data', dir='/tmp') - try: - npu_dump_path = os.path.join(data_path, 'npu_dump.json') - bench_dump_path = os.path.join(data_path, 'bench_dump.json') - npu_stack_path = os.path.join(data_path, 'npu_stack.json') - - with open(npu_dump_path, 'w') as n_d_f: - json.dump(npu_json_data, n_d_f) - with open(bench_dump_path, 'w') as b_d_f: - json.dump(bench_json_data, b_d_f) - with open(npu_stack_path, 'w') as n_s_f: - json.dump({}, n_s_f) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - result_df = ms_comparator.compare_process_custom((npu_dump_path, bench_dump_path, npu_stack_path)) - self.assertListEqual(result_df.values.tolist(), []) - finally: - shutil.rmtree(data_path) - - @patch('msprobe.mindspore.compare.ms_compare.detect_framework_by_dump_json') + @patch('msprobe.mindspore.compare.utils.detect_framework_by_dump_json') def test_check_cross_framework_valid_pytorch(self, mock_detect_framework): mock_detect_framework.return_value = Const.PT_FRAMEWORK @@ -360,7 +208,7 @@ class TestUtilsMethods(unittest.TestCase): self.assertTrue(result) - @patch('msprobe.mindspore.compare.ms_compare.detect_framework_by_dump_json') + @patch('msprobe.mindspore.compare.utils.detect_framework_by_dump_json') def test_check_cross_framework_invalid_framework(self, mock_detect_framework): mock_detect_framework.return_value = Const.MS_FRAMEWORK @@ -368,195 +216,38 @@ class TestUtilsMethods(unittest.TestCase): self.assertFalse(result) - def test_comapre_process(self): - data_path = tempfile.mkdtemp(prefix='dump_data', dir='/tmp') - try: - npu_dump_path = os.path.join(data_path, 'npu_dump.json') - bench_dump_path = os.path.join(data_path, 'bench_dump.json') - npu_stack_path = os.path.join(data_path, 'npu_stack.json') - - npu_data, bench_data, _ = gen_api_mapping_test_data() - with open(npu_dump_path, 'w', encoding='utf8') as n_d_f: - json.dump(npu_data, n_d_f) - with open(bench_dump_path, 'w', encoding='utf8') as b_d_f: - json.dump(bench_data, b_d_f) - with open(npu_stack_path, 'w', encoding='utf8') as n_s_f: - json.dump({}, n_s_f) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(api_mapping=True) - - ms_comparator = MSComparator(mode_config, mapping_config) - result_df = ms_comparator.compare_process((npu_dump_path, bench_dump_path, npu_stack_path)) - self.assertTrue((result_df['Bench Name'] != 'N/A').all()) - finally: - shutil.rmtree(data_path) - - def test_compare_process_with_customize_api_mapping(self): - data_path = tempfile.mkdtemp(prefix='dump_data', dir='/tmp') - try: - npu_dump_path = os.path.join(data_path, 'npu_dump.json') - bench_dump_path = os.path.join(data_path, 'bench_dump.json') - npu_stack_path = os.path.join(data_path, 'npu_stack.json') - user_mapping_path = os.path.join(data_path, 'user_mapping.yaml') - - npu_data, bench_data, user_mapping = gen_api_mapping_test_data(True) - with open(npu_dump_path, 'w', encoding='utf8') as n_d_f: - json.dump(npu_data, n_d_f) - with open(bench_dump_path, 'w', encoding='utf8') as b_d_f: - json.dump(bench_data, b_d_f) - with open(npu_stack_path, 'w', encoding='utf8') as n_s_f: - json.dump({}, n_s_f) - with open(user_mapping_path, 'w', encoding='utf8') as u_m_f: - yaml.safe_dump(user_mapping, u_m_f) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(api_mapping=user_mapping_path) - - ms_comparator = MSComparator(mode_config, mapping_config) - result_df = ms_comparator.compare_process((npu_dump_path, bench_dump_path, npu_stack_path)) - - user_mapping_dict = {} - for i in user_mapping: - user_mapping_dict[i.get('ms_api')] = {'input': i.get('ms_args'), 'output': i.get('ms_output')} - match_set = set() - for key in npu_data.get('data').keys(): - matched_dict = user_mapping_dict.get(key.rsplit('.', 2)[0]) - match_set.update({key + '.input.' + str(i) for i in matched_dict.get('input')}) - match_set.update({key + '.output.' + str(i) for i in matched_dict.get('output')}) - - self.assertTrue((result_df.loc[result_df['NPU Name'].isin(match_set), 'Bench Name'] != 'N/A').all()) - self.assertTrue((result_df.loc[~result_df['NPU Name'].isin(match_set), 'Bench Name'] == 'N/A').all()) - finally: - shutil.rmtree(data_path) - - def test_load_internal_api(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - api_dict = ms_comparator.load_internal_api() - self.assertEqual(api_dict['Functional.abs'], 'Torch.abs') - - def test_process_cell_mapping(self): - self.base_test_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) - self.input_dir = os.path.join(self.base_test_dir, 'resources') - cell_mapping_path = os.path.join(self.input_dir, 'common', 'cell_mapping.yaml') - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.SUMMARY - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(cell_mapping=cell_mapping_path) - - ms_comparator = MSComparator(mode_config, mapping_config) - npu_op_name = ms_comparator.process_cell_mapping(npu_cell_dict.get('op_name')[0]) - self.assertEqual(npu_op_name, 'Module.fc1.Linear.forward.0.input.0') - - def test_read_npy_data(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - - self.temp_file = tempfile.NamedTemporaryFile(suffix='.pt') - tensor = torch.Tensor([1, 2, 3]) - filename = self.temp_file.name.split('/')[-1] - torch.save(tensor, self.temp_file.name) - result = ms_comparator.read_npy_data('/tmp', filename, load_pt_file=True) - self.assertTrue(np.array_equal(result, np.array([1, 2, 3]))) - self.temp_file.close() - - self.temp_file = tempfile.NamedTemporaryFile(suffix='.npy') - tensor = np.array([1, 2, 3]) - filename = self.temp_file.name.split('/')[-1] - np.save(self.temp_file.name, tensor) - result = ms_comparator.read_npy_data('/tmp', filename, load_pt_file=False) - self.assertTrue(np.array_equal(result, np.array([1, 2, 3]))) - self.temp_file.close() - - def test_process_internal_api_mapping(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig(api_mapping=1) - - ms_comparator = MSComparator(mode_config, mapping_config) - - npu_op_name = "Mint.addcmul.0.forward.input.0" - result = ms_comparator.process_internal_api_mapping(npu_op_name) - self.assertEqual(result, "Torch.addcmul.0.forward.input.0") - - npu_op_name = "MintFunctional.addcmul.0.forward.input.0" - result = ms_comparator.process_internal_api_mapping(npu_op_name) - self.assertEqual(result, "Functional.addcmul.0.forward.input.0") - - npu_op_name = "Functional.abs" - result = ms_comparator.process_internal_api_mapping(npu_op_name) - self.assertEqual(result, "Torch.abs") - - def test_get_api_name(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - - api_list = ["Functional", "absolute", "0", "forward", "input", "0"] - result = ms_comparator.get_api_name(api_list) - self.assertEqual(result, "Functional.absolute") - - api_list = ["Mint"] - with self.assertRaises(CompareException): - ms_comparator.get_api_name(api_list) - - def test_process_data_name(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - ms_comparator = MSComparator(mode_config, mapping_config) - - data = pd.DataFrame({ - 'data_name_x': ['A', 'B', 'C'], - 'data_name_y': ['X', 'Y', 'Z'] - }) - - result = ms_comparator.process_data_name(data.copy()) - - expected = pd.DataFrame({ - 'data_name_x': [['A', 'X'], ['B', 'Y'], ['C', 'Z']], - 'data_name_y': ['X', 'Y', 'Z'] - }) - - pd.testing.assert_frame_equal(result, expected) + def test_read_real_data_ms(self): + n_value, b_value = read_real_data(base_dir1, 'numpy_data.npy', base_dir1, 'numpy_data.npy', False) + self.assertTrue(np.array_equal(n_value, np.array([1, 2, 3]))) + self.assertTrue(np.array_equal(b_value, np.array([1, 2, 3]))) + + def test_read_real_data_cross_frame(self): + n_value, b_value = read_real_data(base_dir1, 'numpy_data.npy', base_dir1, 'torch_data.pt', True) + self.assertTrue(np.array_equal(n_value, np.array([1, 2, 3]))) + self.assertTrue(np.array_equal(b_value, np.array([2, 3, 4]))) + + def test_ms_compare(self): + generate_dump_json(base_dir1) + generate_stack_json(base_dir1) + + dump_path = os.path.join(base_dir1, 'dump.json') + + input_param = { + 'npu_json_path': dump_path, + 'bench_json_path': dump_path, + 'is_print_compare_log': True + } + output_path = base_dir1 + + ms_compare(input_param, output_path) + output_files = os.listdir(output_path) + self.assertTrue(any(f.endswith(".xlsx") for f in output_files)) + + input_param2 = { + 'npu_json_path': '', + 'bench_json_path': dump_path, + 'is_print_compare_log': True + } + with self.assertRaises(CompareException) as context: + ms_compare(input_param2, output_path) + self.assertEqual(context.exception.code, 1) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare_utils.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d7fb5e38fb82b309caf3ab2a1b621655d7babc86 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare_utils.py @@ -0,0 +1,24 @@ +import unittest +from unittest.mock import patch + +import numpy as np + +from msprobe.core.common.file_utils import FileCheckConst +from msprobe.mindspore.compare.utils import read_npy_data + + +class TestReadNpyData(unittest.TestCase): + + @patch('msprobe.mindspore.compare.utils.load_npy') + @patch('msprobe.mindspore.compare.utils.FileChecker') + @patch('os.path.join', return_value='/fake/path/to/file.npy') + def test_read_real_data_ms(self, mock_os, mock_file_checker, mock_load_npy): + mock_file_checker.return_value.common_check.return_value = '/fake/path/to/file.npy' + + mock_load_npy.return_value = np.array([1.0, 2.0, 3.0]) + + result = read_npy_data('/fake/dir', 'file_name.npy') + + mock_file_checker.assert_called_once_with('/fake/path/to/file.npy', FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.NUMPY_SUFFIX, False) + mock_load_npy.assert_called_once_with('/fake/path/to/file.npy') + self.assertTrue(np.array_equal(result, np.array([1.0, 2.0, 3.0]))) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_graph_cell_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_graph_cell_dump.py new file mode 100644 index 0000000000000000000000000000000000000000..d3279a3abaaa235b52f676559b4c06ac2cc3718d --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_graph_cell_dump.py @@ -0,0 +1,119 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import re +import unittest +from unittest.mock import MagicMock, patch + +import mindspore as ms +from mindspore import ops + +from msprobe.core.common.const import Const as CoreConst +from msprobe.mindspore.dump.cell_dump_process import cell_construct_wrapper +from msprobe.mindspore.dump.cell_dump_process import rename_filename, sort_filenames, del_same_file +from msprobe.mindspore.dump.cell_dump_process import check_relation + + +class TestCellWrapperProcess(unittest.TestCase): + + @patch('msprobe.mindspore.dump.cell_dump_process.ops.is_tensor') + @patch('msprobe.mindspore.dump.cell_dump_process.td') + @patch('msprobe.mindspore.dump.cell_dump_process.td_in') + def test_cell_construct_wrapper(self, mock_td_in, mock_td, mock_istensor): + + # Mock the TensorDump operations + mock_td.return_value = MagicMock() + mock_td_in.return_value = MagicMock() + mock_istensor.return_value = False + + # Create a mock cell with necessary attributes + mock_cell = MagicMock() + mock_cell.data_mode = "all" + mock_cell.dump_path = "mock_dump_path" + mock_cell.cell_prefix = "mock_cell_prefix" + + # Define a mock function to wrap + def mock_func(*args, **kwargs): + return args + + # Wrap the mock function using cell_construct_wrapper + wrapped_func = cell_construct_wrapper(mock_func, mock_cell) + + # Create mock inputs + mock_input = ms.Tensor([1, 2, 3]) + mock_args = (mock_input,) + + # Call the wrapped function + wrapped_func(mock_cell, *mock_args) + + # Verify that the TensorDump operations were not called + mock_td_in.assert_not_called() + mock_td.assert_not_called() + + +class TestSortFilenames(unittest.TestCase): + + @patch('os.listdir') + def test_sort_filenames(self, mock_listdir): + # Mock the list of filenames returned by os.listdir + mock_listdir.return_value = [ + 'Cell.network._backbone.model.LlamaModel.backward.0.input.0_float16_177.npy', + 'Cell.network._backbone.model.LlamaModel.forward.0.input.0_in_int32_1.npy', + 'Cell.network._backbone.model.LlamaModel.forward.0.output.10_float16_165.npy', + 'Cell.network._backbone.model.norm_out.LlamaRMSNorm.backward.0.input.0_float16_178.npy' + ] + + # Mock the CoreConst values + CoreConst.REPLACEMENT_CHARACTER = '_' + CoreConst.NUMPY_SUFFIX = '.npy' + + # Expected sorted filenames + expected_sorted_filenames = [ + 'Cell.network._backbone.model.LlamaModel.forward.0.input.0_in_int32_1.npy', + 'Cell.network._backbone.model.LlamaModel.forward.0.output.10_float16_165.npy', + 'Cell.network._backbone.model.LlamaModel.backward.0.input.0_float16_177.npy', + 'Cell.network._backbone.model.norm_out.LlamaRMSNorm.backward.0.input.0_float16_178.npy' + ] + + # Call the function + sorted_filenames = sort_filenames('/mock/path') + + # Assert the filenames are sorted correctly + self.assertEqual(sorted_filenames, expected_sorted_filenames) + + +class TestCheckRelation(unittest.TestCase): + + def setUp(self): + CoreConst.SEP = '.' + global KEY_LAYERS + KEY_LAYERS = "layers" + + def test_direct_parent_child_relation(self): + self.assertTrue(check_relation("network._backbone", "network")) + self.assertTrue(check_relation("network._backbone.model", "network._backbone")) + + def test_no_relation(self): + self.assertFalse(check_relation("network._backbone", "network.loss")) + self.assertFalse(check_relation("network._backbone.model", "network.loss")) + + def test_layer_pattern_relation(self): + self.assertTrue(check_relation("network.model.layers.0", "network.model")) + self.assertTrue(check_relation("network._backbone.model.layers.1", "network._backbone.model")) + + def test_edge_cases(self): + self.assertFalse(check_relation("", "network")) + self.assertFalse(check_relation("network.layer1", "")) + self.assertFalse(check_relation("", "")) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_ms_debugger_config.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_ms_debugger_config.py index 033b0c1ea5769c3f1f8e19dd8b45c48918e15814..8a7195eac824485e75d8c1ba0752715c7c6a5600 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_ms_debugger_config.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_ms_debugger_config.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,14 +17,17 @@ import unittest from unittest.mock import patch from msprobe.core.common.const import Const -from msprobe.core.common_config import CommonConfig, BaseConfig +from msprobe.core.common.log import logger +from msprobe.core.common_config import CommonConfig from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.mindspore.ms_config import StatisticsConfig class TestDebuggerConfig(unittest.TestCase): + @patch.object(logger, "error") @patch("msprobe.mindspore.debugger.debugger_config.create_directory") - def test_init(self, _): + def test_init(self, _, mock_logger_error): json_config = { "dump_path": "/absolute_path", "rank": [], @@ -32,12 +35,13 @@ class TestDebuggerConfig(unittest.TestCase): "level": "L2" } common_config = CommonConfig(json_config) - task_config = BaseConfig(json_config) + task_config = StatisticsConfig(json_config) debugger_config = DebuggerConfig(common_config, task_config) self.assertEqual(debugger_config.task, Const.STATISTICS) self.assertEqual(debugger_config.file_format, "npy") self.assertEqual(debugger_config.check_mode, "all") self.assertEqual(debugger_config.overflow_nums, 1) + self.assertEqual(debugger_config.tensor_list, []) common_config.level = "L1" common_config.task = Const.FREE_BENCHMARK @@ -49,17 +53,16 @@ class TestDebuggerConfig(unittest.TestCase): task_config.handler_type = FreeBenchmarkConst.FIX task_config.pert_mode = FreeBenchmarkConst.ADD_NOISE - with self.assertRaises(Exception) as context: + with self.assertRaises(ValueError): DebuggerConfig(common_config, task_config) - self.assertEqual(str(context.exception), - "pert_mode must be improve_precision or empty when handler_type is fix, " - f"but got {FreeBenchmarkConst.ADD_NOISE}.") + mock_logger_error.assert_called_with("pert_mode must be improve_precision or empty when handler_type is fix, " + f"but got {FreeBenchmarkConst.ADD_NOISE}.") + mock_logger_error.reset_mock() task_config.handler_type = FreeBenchmarkConst.FIX task_config.pert_mode = FreeBenchmarkConst.DEFAULT_PERT_TYPE task_config.fuzz_stage = Const.BACKWARD - with self.assertRaises(Exception) as context: + with self.assertRaises(ValueError): DebuggerConfig(common_config, task_config) - self.assertEqual(str(context.exception), - "handler_type must be check or empty when fuzz_stage is backward, " - f"but got {task_config.handler_type}.") + mock_logger_error.assert_called_with("handler_type must be check or empty when fuzz_stage is backward, " + f"but got {task_config.handler_type}.") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_ms_precision_debugger.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_ms_precision_debugger.py index 86fdcc08385fc0fe68da541b473a02e130d28478..dd8259c62437216ccfad804897f6b1abc5ad8359 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_ms_precision_debugger.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_ms_precision_debugger.py @@ -16,13 +16,14 @@ import unittest from unittest.mock import patch, MagicMock -from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.const import Const, MsgConst +from msprobe.core.common_config import CommonConfig from msprobe.mindspore.cell_processor import CellProcessor from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell +from msprobe.mindspore.ms_config import StatisticsConfig from msprobe.mindspore.runtime import Runtime @@ -48,7 +49,7 @@ class TestPrecisionDebugger(unittest.TestCase): } common_config = CommonConfig(json_config) - task_config = BaseConfig(json_config) + task_config = StatisticsConfig(json_config) handler = Handler() mock_get_mode = MagicMock() @@ -74,7 +75,7 @@ class TestPrecisionDebugger(unittest.TestCase): debugger.start() service = mock_Service.return_value mock_Service.assert_called_with(debugger.config) - service.start.assert_called_with(None) + service.start.assert_called_with(None, None) PrecisionDebugger._instance = None with self.assertRaises(Exception) as context: diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/handler/test_ms_base_handler.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/handler/test_ms_base_handler.py index d7f5b0745cff481d0bc2e5771df36beb492d4015..91230456911fcace6877869a270264b6b95b6793 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/handler/test_ms_base_handler.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/handler/test_ms_base_handler.py @@ -24,6 +24,7 @@ from msprobe.mindspore.common.log import logger from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams from msprobe.mindspore.free_benchmark.common.utils import Tools from msprobe.mindspore.free_benchmark.handler.base_handler import BaseHandler +from msprobe.mindspore.dump.hook_cell.api_register import get_api_register class Handler(BaseHandler): @@ -45,6 +46,7 @@ class TestBaseHandler(unittest.TestCase): @classmethod def setUpClass(cls): cls.base_handler = Handler("api_name_with_id") + get_api_register(True).restore_all_api() def test___init__(self): base_handler = Handler("api_name_with_id") @@ -93,7 +95,7 @@ class TestBaseHandler(unittest.TestCase): first_tensor = Tensor([1.0, 1.2], dtype=ms.bfloat16) second_tensor = Tensor([1.5, 2.0], dtype=ms.bfloat16) - target = ops.max(ops.div(second_tensor.to(ms.float32), first_tensor.to(ms.float32)))[0].item() + target = ops.max(ops.div(ops.cast(second_tensor, ms.float32), ops.cast(first_tensor, ms.float32)))[0].item() ret = self.base_handler.get_endless_norm(first_tensor, second_tensor, abs_tol) self.assertEqual(ret, target) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/perturbation/test_ms_perturbation_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/perturbation/test_ms_perturbation_factory.py index 858e664bbaddb3506bf53ea067eeca1c9706b43b..a4458912149fc8600d32a542c398a335be5d636d 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/perturbation/test_ms_perturbation_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/perturbation/test_ms_perturbation_factory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +14,9 @@ # limitations under the License. import unittest +from unittest.mock import patch +from msprobe.mindspore.common.log import logger from msprobe.mindspore.free_benchmark.perturbation.perturbation_factory import PerturbationFactory from msprobe.mindspore.free_benchmark.common.config import Config from msprobe.mindspore.common.const import FreeBenchmarkConst @@ -27,14 +29,14 @@ from msprobe.mindspore.free_benchmark.perturbation.exchange_value import Exchang class TestPerturbationFactory(unittest.TestCase): - def test_create(self): + @patch.object(logger, "error") + def test_create(self, mock_logger_error): api_name = "Functional.add.0" Config.pert_type = "UNKNOWN" - with self.assertRaises(Exception) as context: + with self.assertRaises(ValueError): PerturbationFactory.create(api_name) - self.assertEqual(str(context.exception), - "UNKNOWN is a invalid perturbation type") + mock_logger_error.assert_called_with("UNKNOWN is a invalid perturbation type") Config.pert_type = FreeBenchmarkConst.EXCHANGE_VALUE pert = PerturbationFactory.create(api_name) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/test_ms_api_pynative_self_check.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/test_ms_api_pynative_self_check.py index c4482a22f042723f12431b56c730a8d69957b63c..e07417aba8c745833a3f551a9e0489d848a58bb1 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/test_ms_api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/test_ms_api_pynative_self_check.py @@ -99,11 +99,13 @@ class TestApiPyNativeSelfCheck(TestCase): _, forward_hook, backward_hook, _ = self.checker.build_hook("Functional.add.") cell = Cell() + cell.msprobe_input_kwargs = {} with patch("msprobe.mindspore.free_benchmark.api_pynative_self_check.need_wrapper_func", return_value=False): self.assertIsNone(forward_hook(cell, "input", "output")) cell = Cell() + cell.msprobe_input_kwargs = {} self.checker.api_list = ["mindspore.ops.add"] self.checker.ori_func["mindspore.ops.add"] = "add" with patch("msprobe.mindspore.free_benchmark.api_pynative_self_check.need_wrapper_func", return_value=True), \ diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/test_ms_self_check_tool_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/test_ms_self_check_tool_factory.py index fa68b8896c26d4156833c54d2b2bf5b443164e8f..4f3ddd45b5a05162c60abe831967dd449f3f5ae6 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/test_ms_self_check_tool_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/test_ms_self_check_tool_factory.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,11 +12,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" import os import unittest +from unittest.mock import patch +from msprobe.core.common.log import logger from msprobe.mindspore.free_benchmark.self_check_tool_factory import SelfCheckToolFactory from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelfCheck from msprobe.mindspore.debugger.debugger_config import DebuggerConfig @@ -28,7 +28,8 @@ from msprobe.core.common.const import Const class TestSelfCheckToolFactory(unittest.TestCase): - def test_create(self): + @patch.object(logger, "error") + def test_create(self, mock_logger_error): common_config = CommonConfig({}) common_config.task = Const.FREE_BENCHMARK common_config.dump_path = os.path.dirname(os.path.realpath(__file__)) @@ -36,16 +37,16 @@ class TestSelfCheckToolFactory(unittest.TestCase): config = DebuggerConfig(common_config, task_config) config.level = "UNKNOWN" - with self.assertRaises(Exception) as context: + with self.assertRaises(ValueError): SelfCheckToolFactory.create(config) - self.assertEqual(str(context.exception), "UNKNOWN is not supported.") + mock_logger_error.assert_called_with("UNKNOWN is not supported.") + mock_logger_error.reset_mock() config.level = MsConst.API config.execution_mode = MsConst.GRAPH_KBYK_MODE - with self.assertRaises(Exception) as context: + with self.assertRaises(ValueError): SelfCheckToolFactory.create(config) - self.assertEqual(str(context.exception), - f"Task free_benchmark is not supported in this mode: {MsConst.GRAPH_KBYK_MODE}.") + mock_logger_error.assert_called_with(f"Task free_benchmark is not supported in this mode: {MsConst.GRAPH_KBYK_MODE}.") config.execution_mode = MsConst.PYNATIVE_MODE tool = SelfCheckToolFactory.create(config) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_analyzer.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_analyzer.py index 802769d9005916c8723d436349d13ca7f557a00a..af8f6b0477f766507db55dbe345f2d802415dc14 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_analyzer.py @@ -1,6 +1,7 @@ import os import shutil import json +import time import numpy as np import mindspore as ms from unittest import TestCase, mock @@ -15,7 +16,8 @@ class TestGradAnalyzer(TestCase): @classmethod def setUpClass(cls): cls.output_path = "./test_output" - cls.dump_dir = f"{cls.output_path}/rank0/Dump" + cls.time_stamp = str(int(time.time())) + cls.dump_dir = f"{cls.output_path}/rank0/Dump{cls.time_stamp}" cls.save_dir = f"{cls.output_path}/rank0" os.makedirs(cls.dump_dir, exist_ok=True) @@ -31,7 +33,8 @@ class TestGradAnalyzer(TestCase): 'get_context.side_effect': lambda x: { GradConst.OUTPUT_PATH: self.output_path, GradConst.LEVEL: GradConst.LEVEL2, - GradConst.BOUNDS: [-0.1, 0.0, 0.1] + GradConst.BOUNDS: [-0.1, 0.0, 0.1], + GradConst.TIME_STAMP: self.time_stamp, }[x] })) # Clear dump directory before each test diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/ms_monitor/test_common_func.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/ms_monitor/test_common_func.py new file mode 100644 index 0000000000000000000000000000000000000000..d0753c5c2300b58814504f1e2a0e1bfe7cb56e12 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/ms_monitor/test_common_func.py @@ -0,0 +1,120 @@ +import pytest +from unittest.mock import patch, MagicMock +from mindspore import nn, context +from mindspore.common.initializer import Normal +import mindspore as ms + +from msprobe.mindspore.monitor.common_func import ( + is_valid_instance, + get_submodules, + get_parameters, + get_rank, + comm_is_initialized, + optimizer_pre_hook, + optimizer_post_hook +) + +TORCH_AVAILABLE = False +try: + import torch + import torch.nn as torch_nn + TORCH_AVAILABLE = True +except ImportError: + TORCH_AVAILABLE = False + + +class TestModelUtils: + @classmethod + def setup_class(cls): + """Setup once for all tests in this class""" + cls.ms_model = MSModel() + if TORCH_AVAILABLE: + cls.torch_model = TorchModel() + + @classmethod + def teardown_class(cls): + """Cleanup after all tests in this class""" + pass + + + def test_is_valid_instance_if_model_is_cell_or_module_then_return_true(self): + with patch('msprobe.mindspore.monitor.common_func.is_mindtorch') as mock_is_mindtorch: + if TORCH_AVAILABLE: + mock_is_mindtorch.return_value = True + assert is_valid_instance(self.torch_model) + mock_is_mindtorch.return_value = False + assert is_valid_instance(self.ms_model) + + def test_is_valid_instance_if_input_is_string_then_return_false(self): + assert not is_valid_instance("not a model") + + def test_is_valid_instance_if_input_is_number_then_return_false(self): + assert not is_valid_instance(123) + + def test_get_submodules_if_model_is_valid_then_return_non_empty_dict(self): + with patch('msprobe.mindspore.monitor.common_func.is_mindtorch') as mock_is_mindtorch: + mock_is_mindtorch.return_value = True + if TORCH_AVAILABLE: + submodules = dict(get_submodules(self.torch_model)) + assert len(submodules) > 0 + assert any(name == 'conv1' for name in submodules) + + mock_is_mindtorch.return_value = False + submodules = dict(get_submodules(self.ms_model)) + assert len(submodules) > 0 + assert any(name.endswith('conv1') for name in submodules) + + + def test_get_submodules_if_model_is_invalid_then_return_empty_dict(self): + assert get_submodules("invalid") == {} + + def test_get_parameters_if_model_is_valid_then_return_non_empty_dict(self): + with patch('msprobe.mindspore.monitor.common_func.is_mindtorch') as mock_is_mindtorch: + mock_is_mindtorch.return_value = True + if TORCH_AVAILABLE: + params = dict(get_parameters(self.torch_model)) + assert any(name == 'conv1.weight' for name in params) + mock_is_mindtorch.return_value = False + params = dict(get_parameters(self.ms_model)) + assert any('conv1.weight' in name for name in params) + + + def test_get_parameters_if_model_is_invalid_then_return_empty_dict(self): + assert get_parameters(123) == {} + + def test_get_rank_if_comm_initialized_then_return_integer(self): + rank = get_rank() + assert isinstance(rank, int) + assert rank >= 0 + + def test_comm_is_initialized_when_called_then_return_boolean(self): + assert isinstance(comm_is_initialized(), bool) + + +# Test models +class MSModel(nn.Cell): + def __init__(self): + super().__init__() + self.conv1 = nn.Conv2d(3, 64, 3, has_bias=True, weight_init=Normal(0.02)) + self.bn1 = nn.BatchNorm2d(64) + self.relu = nn.ReLU() + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + return x + +if TORCH_AVAILABLE: + class TorchModel(torch_nn.Module): + def __init__(self): + super().__init__() + self.conv1 = torch_nn.Conv2d(3, 64, 3) + self.bn1 = torch_nn.BatchNorm2d(64) + self.relu = torch_nn.ReLU() + + def forward(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + return x \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/ms_monitor/test_opt_collect.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/ms_monitor/test_opt_collect.py new file mode 100644 index 0000000000000000000000000000000000000000..8471c289dce975754a957e2efad2878af1fd588e --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/ms_monitor/test_opt_collect.py @@ -0,0 +1,223 @@ +import pytest +import numpy as np +from mindspore import Tensor, nn, ops +from unittest.mock import MagicMock, patch + +from msprobe.core.common.const import MonitorConst +# Import the classes to test +from msprobe.core.common.log import logger +from msprobe.mindspore.monitor.optimizer_collect import ( + OptimizerMon, + MixPrecisionOptimizerMon, + MegatronDistributedOptimizerMon, + MegatronChainedDistributedOptimizerMon, + MegatronChainedMixPrecisionOptimizerMon, + DeepSpeedZeroOptimizerMon, + DeepSpeedZeroOptimizerStage0Mon, + DeepSpeedZeroOptimizerStage1or2Mon, + DeepSpeedZeroOptimizerStage3Mon, + OptimizerMonFactory +) + +class TestOptimizerMon: + @classmethod + def setup_class(cls): + """Setup once for all tests in this class""" + cls.mock_monitor = MagicMock() + cls.mock_monitor.name2tag = {"test_param": {MonitorConst.POST_GRAD: "test_tag"}} + cls.mock_monitor.duplicate_param = {} + cls.mock_monitor.params_have_main_grad = False + cls.mock_monitor.fsdp_wrapped_module = False + cls.mock_monitor.mv_distribution = True + cls.mock_monitor.mg_direction = True + cls.mock_monitor.ur_distribution = True + cls.mock_monitor.update_heatmap_visualizer = {"test_param": MagicMock()} + cls.mock_monitor.ratio_heatmap_visualizer = {"test_param": MagicMock()} + + def test_fetch_grad_if_param_has_valid_grad_then_return_correct_grad_values(self): + # Setup + param = MagicMock() + expected_grad = Tensor([1.0, 2.0, 3.0]) + param.grad = expected_grad + params2name = {param: "test_param"} + optimizer = MagicMock() + mon = OptimizerMon(optimizer) + + # Execute + result = mon.fetch_grad(self.mock_monitor, params2name) + + # Verify + assert len(result) == 1 + assert (result["test_tag"] == expected_grad).all() + self.mock_monitor.register_param_call_id.assert_called_once_with("hook_optimizer", "test_tag") + + def test_fetch_grad_if_param_has_main_grad_then_return_main_grad_values(self): + # Setup + param = MagicMock() + expected_grad = Tensor(np.array([1.5, 2.5])) + param.main_grad = expected_grad + param.grad = None + params2name = {param: "test_param"} + optimizer = MagicMock() + self.mock_monitor.params_have_main_grad = True + mon = OptimizerMon(optimizer) + + # Execute + result = mon.fetch_grad(self.mock_monitor, params2name) + + # Verify + assert len(result) == 1 + assert (result["test_tag"] == expected_grad).all() + + def test_fetch_mv_if_state_complete_then_return_correct_momentum_values(self): + # Setup + param = MagicMock() + params2name = {param: "test_param"} + optimizer = MagicMock() + optimizer.state = { + param: { + "exp_avg": Tensor([0.1]), + "exp_avg_sq": Tensor([0.2]), + "step": 10 + } + } + optimizer.defaults = {'betas': (0.9, 0.999), 'eps': 1e-8} + optimizer.param_groups = [{}] + + mon = OptimizerMon(optimizer) + mon.fp16_to_fp32_param = {} + + # Execute + exp_avg, exp_avg_sq, update, ratio = mon.fetch_mv(self.mock_monitor, params2name) + + # Verify + beta1, beta2 = optimizer.defaults['betas'] + step = optimizer.state[param]['step'] + + expected_exp_avg_hat = 0.1 / (1 - beta1**step) + expected_exp_avg_sq_hat = 0.2 / (1 - beta2**step) + expected_update = expected_exp_avg_hat / (np.sqrt(expected_exp_avg_sq_hat) + optimizer.defaults['eps']) + expected_ratio = expected_exp_avg_hat / np.sqrt(expected_exp_avg_sq_hat) + + assert exp_avg["test_param"] == Tensor([0.1]) + assert exp_avg_sq["test_param"] == Tensor([0.2]) + assert update["test_param"] == Tensor([expected_update]) + assert ratio["test_param"] == Tensor([expected_ratio]) + + def test_narrow_from_flatten_if_state_not_partitioned_then_return_original_state(self): + # Setup + param = MagicMock() + flatten_state = Tensor([1.0, 2.0, 3.0]) + mon = OptimizerMon(MagicMock()) + + # Execute + result = mon.narrow_from_flatten(param, flatten_state) + + # Verify + assert (result == flatten_state).all() + +class TestMixPrecisionOptimizerMon: + @classmethod + def setup_class(cls): + cls.mock_monitor = MagicMock() + cls.mock_monitor.mv_distribution = True + cls.mock_monitor.mg_direction = True + cls.mock_monitor.ur_distribution = True + cls.mock_monitor.update_heatmap_visualizer = {'param1': MagicMock(), 'param2': MagicMock()} + cls.mock_monitor.ratio_heatmap_visualizer = {'param1': MagicMock(), 'param2': MagicMock()} + + def test_map_fp16_to_fp32_param_if_multiple_groups_then_create_correct_mappings(self): + # Setup + optimizer = MagicMock() + fp16_params = [MagicMock(), MagicMock(), MagicMock()] + fp32_params = [MagicMock(), MagicMock(), MagicMock()] + optimizer.float16_groups = [fp16_params[:2], [fp16_params[2]]] + optimizer.fp32_from_float16_groups = [fp32_params[:2], [fp32_params[2]]] + + mon = MixPrecisionOptimizerMon(optimizer) + + # Execute + mon.map_fp16_to_fp32_param(optimizer) + + # Verify + assert len(mon.fp16_to_fp32_param) == 3 + for fp16, fp32 in zip(fp16_params, fp32_params): + assert mon.fp16_to_fp32_param[fp16] == fp32 + +class TestDeepSpeedZeroOptimizerStage1or2Mon: + @classmethod + def setup_class(cls): + """Setup once for all tests in this class""" + cls.mock_monitor = MagicMock() + cls.mock_monitor.name2tag = {"test_param": {MonitorConst.POST_GRAD: "test_tag"}} + cls.mock_monitor.duplicate_param = {} + cls.mock_monitor.params_have_main_grad = False + cls.mock_monitor.mg_direction = True + cls.mock_monitor.ur_distribution = True + + def test_fetch_grad_if_param_in_partition_then_return_correct_grad_slice(self): + # Setup + optimizer = MagicMock() + param = MagicMock() + params2name = {param: "test_param"} + expected_grad = Tensor(np.array([1.0, 2.0, 3.0])) + param.main_grad = expected_grad + param.grad = None + optimizer.bit16_groups = [[param]] + optimizer.cpu_offload = False + mon = DeepSpeedZeroOptimizerStage1or2Mon(optimizer) + mon.param2group = {param: 0} + mon.get_param_index = MagicMock(return_value=1) + mon.param_not_in_partition = MagicMock(return_value=False) + mon.get_position = MagicMock(return_value=(3, 3)) # start at index 3, length 3 + + # MagicMock the averaged_gradients structure + optimizer.averaged_gradients = { + 0: [ + None, # index 0 + Tensor(np.array([1.0, 2.0, 3.0])) # index 1 + ] + } + + # Execute + result = mon.fetch_grad(self.mock_monitor, params2name) + + # Verify + assert len(result) == 1 + assert (result["test_tag"] == expected_grad).all() + +class TestOptimizerMonFactory: + @classmethod + def setup_class(cls): + cls.mock_monitor = MagicMock() + cls.mock_monitor.mv_distribution = True + cls.mock_monitor.mg_direction = True + cls.mock_monitor.ur_distribution = True + cls.mock_monitor.update_heatmap_visualizer = {'param1': MagicMock(), 'param2': MagicMock()} + cls.mock_monitor.ratio_heatmap_visualizer = {'param1': MagicMock(), 'param2': MagicMock()} + + def test_create_optimizer_mon_if_chained_optimizer_then_return_correct_monitor_type(self): + # Setup + base_optimizer = MagicMock() + base_optimizer.__class__.__name__ = "DistributedOptimizer" + optimizer = MagicMock() + optimizer.__class__.__name__ = "ChainedOptimizer" + optimizer.chained_optimizers = [base_optimizer] + + # Execute + result = OptimizerMonFactory.create_optimizer_mon(optimizer) + + # Verify + assert isinstance(result, MegatronChainedDistributedOptimizerMon) + + def test_create_optimizer_mon_if_deepspeed_stage3_then_return_stage3_monitor(self): + # Setup + optimizer = MagicMock() + optimizer.__class__.__name__ = "DeepSpeedZeroOptimizer_Stage3" + + # Execute + result = OptimizerMonFactory.create_optimizer_mon(optimizer) + + # Verify + assert isinstance(result, DeepSpeedZeroOptimizerStage3Mon) + assert result.stage == '3' diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/save/test_debugger_save_mindspore.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/save/test_debugger_save_mindspore.py new file mode 100644 index 0000000000000000000000000000000000000000..fcefbb8c339ad6de1d14eaae7f75e6947efc5196 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/save/test_debugger_save_mindspore.py @@ -0,0 +1,364 @@ +import unittest +import os +import json +import mindspore +import numpy as np +import shutil +from unittest.mock import patch + +from msprobe.mindspore import PrecisionDebugger +from msprobe.core.data_dump.data_processor.mindspore_processor import MindsporeDataProcessor +from msprobe.mindspore.dump.hook_cell.api_register import get_api_register + +current_file = __file__ +parent_dir = os.path.abspath(os.path.dirname(current_file)) +test_dir = os.path.join(parent_dir, "test_dir") + +def deep_compare(obj1, obj2, float_tolerance=1e-5): + """ + Recursively compare two objects to check if they are the same. + Supports nested dictionaries and lists. + """ + if type(obj1) != type(obj2): + return False + if isinstance(obj1, dict): + if obj1.keys() != obj2.keys(): + return False + return all(deep_compare(obj1[key], obj2[key]) for key in obj1) + if isinstance(obj1, (tuple, list)): + if len(obj1) != len(obj2): + return False + return all(deep_compare(item1, item2) for item1, item2 in zip(obj1, obj2)) + if isinstance(obj1, (int, float)): + return abs(obj1 - obj2) < float_tolerance + return obj1 == obj2 + +class TestDebuggerSave(unittest.TestCase): + @staticmethod + def write_config_json(step, async_dump, mode, dump_path, config_file_path): + task = "tensor" if mode == "tensor" else "statistics" + statistics_summary_mode = "statistics" if mode == "statistics" else "md5" + config = { + "task": task, + "dump_path": dump_path, + "rank": [], + "step": step, + "level": "debug", + "enable_dataloader": False, + "async_dump": async_dump, + "statistics": { + "summary_mode": statistics_summary_mode, + } + } + with open(config_file_path, "w", encoding="utf-8") as f: + json.dump(config, f, indent=4, ensure_ascii=False) + + @staticmethod + def read_debug_json_into_dict(debug_json_path): + with open(debug_json_path, "r", encoding="utf-8") as f: + debug_json = json.load(f) + return debug_json + + @staticmethod + def check_real_npy(npy_path, target_ms_tensor, check_values=True, rtol=1e-5, atol=1e-8): + """ + Enhanced version with optional value comparison. + + Args: + npy_path (str): Path to the .npy file + target_ms_tensor: Target mindspore tensor to compare + check_values (bool): If True, also compare array values + rtol, atol: Relative and absolute tolerances for value comparison + + Returns: + bool: True if all checks pass + """ + # Convert mindspore tensor to numpy if needed + if hasattr(target_ms_tensor, 'numpy'): + target_ms_tensor = target_ms_tensor.numpy() + # Load the npy file + try: + npy_data = np.load(npy_path) + except FileNotFoundError: + print(f"Error: The file {npy_path} does not exist.") + return False + except Exception as e: + print(f"Error loading npy file: {e}") + return False + # Check shapes + if npy_data.shape != target_ms_tensor.shape: + print(f"Shape mismatch: npy data shape is {npy_data.shape}, target tensor shape is {target_ms_tensor.shape}") + return False + # Check dtypes + if npy_data.dtype != target_ms_tensor.dtype: + print(f"Shape mismatch: npy data dtype is {npy_data.dtype}, target tensor dtype is {target_ms_tensor.dtype}") + return False + # Optionally check values + if check_values: + if not np.allclose(npy_data, target_ms_tensor, rtol=rtol, atol=atol): + print("Value mismatch: npy data and target tensor values do not match within the specified tolerances.") + return False + + return True + + def setUp(self): + if not os.path.exists(test_dir): + os.makedirs(test_dir) + PrecisionDebugger._instance = None + self.original_mindspore_special_type = MindsporeDataProcessor.mindspore_special_type + MindsporeDataProcessor.mindspore_special_type = tuple([mindspore.Tensor]) + + def tearDown(self): + if os.path.exists(test_dir): + shutil.rmtree(test_dir) + PrecisionDebugger._instance = None + MindsporeDataProcessor.mindspore_special_type = self.original_mindspore_special_type + get_api_register(True).restore_all_api() + + @patch("msprobe.mindspore.debugger.precision_debugger.set_register_backward_hook_functions") + def test_save_real_tensor(self, _): + data = {"a": mindspore.Tensor([1., 2.])} + step = [] + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + + # check npy file + npy_path = os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "data_dict.0.debug.a.npy") + assert self.check_real_npy(npy_path, data["a"]) + + # check debug json + target_debug_info = { + "a": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "data_name": "data_dict.0.debug.a.npy" + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + @patch("msprobe.mindspore.debugger.precision_debugger.set_register_backward_hook_functions") + def test_save_md5(self, _): + data = {"a": mindspore.Tensor([1., 2.])} + step = [] + async_dump = False + mode = "md5" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check debug json + target_debug_info = { + "a": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "md5": "2e3fa576" + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + @patch("msprobe.mindspore.debugger.precision_debugger.set_register_backward_hook_functions") + def test_save_multiple_steps(self, _): + data = {"a": mindspore.Tensor([1., 2.])} + step = [0, 1, 2] + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + for _ in step: + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check npy file + for i in step: + npy_path = os.path.join(dump_path, f"step{i}", "rank", "dump_tensor_data", "data_dict.0.debug.a.npy") + assert self.check_real_npy(npy_path, data["a"]) + # check debug json + target_debug_info = { + "a": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "data_name": "data_dict.0.debug.a.npy" + } + } + for i in step: + debug_json_path = os.path.join(dump_path, f"step{i}", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + @patch("msprobe.mindspore.debugger.precision_debugger.set_register_backward_hook_functions") + def test_async_save_tensor(self, _): + data = {"a": mindspore.Tensor([1., 2.])} + step = [] + async_dump = True + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check npy file + npy_path = os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "data_dict.0.debug.a.npy") + assert self.check_real_npy(npy_path, data["a"]) + # check debug json + target_debug_info = { + "a": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2 + ], + "data_name": "data_dict.0.debug.a.npy", + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302 + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + @patch("msprobe.mindspore.debugger.precision_debugger.set_register_backward_hook_functions") + def test_async_save_md5(self, _): + # async_dump case, md5 configuration not working,only save statistics + data = {"a": mindspore.Tensor([1., 2.])} + step = [] + async_dump = True + mode = "md5" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check debug json + target_debug_info = { + "a": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302 + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + @patch("msprobe.mindspore.debugger.precision_debugger.set_register_backward_hook_functions") + def test_save_multiple_times(self, _): + data = {"a": mindspore.Tensor([1., 2.])} + step = [] + call_times = 3 + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + for _ in range(call_times): + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check npy file + for i in range(call_times): + npy_path = os.path.join(dump_path, "step0", "rank", "dump_tensor_data", f"data_dict.{i}.debug.a.npy") + assert self.check_real_npy(npy_path, data["a"]) + # check debug json + for i in range(call_times): + target_debug_info = { + "a": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "data_name": f"data_dict.{i}.debug.a.npy" + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"][f"data_dict.{i}.debug"], target_debug_info) + + @patch("msprobe.mindspore.debugger.precision_debugger.set_register_backward_hook_functions") + def test_save_compilcated_data_structure(self, _): + x = mindspore.Tensor([1., 2.]) + complicated_structure = [{"a_key": x}] + step = [] + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(complicated_structure, "complicated_structure") + PrecisionDebugger.step() + complicated_structure_info_list = [ + x, + os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "complicated_structure.0.debug.0.a_key.npy"), + "complicated_structure.0.debug", + [ + { + "a_key": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "data_name": "complicated_structure.0.debug.0.a_key.npy" + } + } + ], + ] + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + target_tensor, target_tensor_path, target_tensor_key, target_tensor_info = complicated_structure_info_list + assert self.check_real_npy(target_tensor_path, target_tensor) + assert deep_compare(debug_json_dict["data"][target_tensor_key], target_tensor_info) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_cell_processor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_cell_processor.py index 40f5c0164115e18cdd49c046ce29967e7a3f63eb..8027034dec9398ec6be4261d0b0af8078b4ed9ec 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_cell_processor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_cell_processor.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,132 +16,342 @@ import unittest from unittest.mock import MagicMock, patch +import mindspore as ms +from mindspore import Tensor +from mindspore.ops.operations import _inner_ops + from msprobe.core.common.const import Const +from msprobe.core.common.exceptions import MsprobeException from msprobe.core.data_dump.scope import ModuleRangeScope -from msprobe.mindspore.cell_processor import CellProcessor - - -class MockCell: - def __init__(self): - self.mindstudio_reserved_name = None +from msprobe.mindspore.cell_processor import CellProcessor, get_cell_construct +from msprobe.mindspore.common.log import logger class TestCellProcessor(unittest.TestCase): + @classmethod + def setUpClass(cls): + CellProcessor.reset_cell_stats() + cls.scope = MagicMock(spec=ModuleRangeScope) + cls.processor = CellProcessor(cls.scope) - def setUp(self): - # 重置静态变量 + @classmethod + def tearDownClass(cls): CellProcessor.reset_cell_stats() - self.scope = MagicMock(spec=ModuleRangeScope) - self.processor = CellProcessor(self.scope) - def test_init_with_module_range_scope(self): - self.assertIsInstance(self.processor.scope, ModuleRangeScope) + def test_class_attribute(self): + self.assertTrue(hasattr(CellProcessor, 'cell_count')) + self.assertTrue(hasattr(CellProcessor, 'cell_stack')) + self.assertTrue(hasattr(CellProcessor, 'api_parent_node')) + self.assertTrue(hasattr(CellProcessor, 'module_node')) + self.assertTrue(hasattr(CellProcessor, 'cell_bw_hook_kernels')) + self.assertTrue(hasattr(CellProcessor, 'cell_backward_pre_hook')) + self.assertTrue(hasattr(CellProcessor, 'cell_backward_hook')) - def test_init_with_none_scope(self): + def test__init(self): + self.assertIsInstance(self.processor.scope, ModuleRangeScope) processor = CellProcessor(None) self.assertIsNone(processor.scope) - def test_set_cell_count_new_cell(self): - count = self.processor.set_cell_count("cell1") + def test_get_cell_construct(self): + def construct(self, *args, **kwargs): + return len(args) + + _constrct = get_cell_construct(construct) + ret = _constrct(self, 'argument') + self.assertFalse(hasattr(self, 'msprobe_input_kwargs')) + self.assertEqual(ret, 1) + + setattr(self, 'msprobe_hook', True) + _constrct = get_cell_construct(construct) + ret = _constrct(self, 'argument') + self.assertEqual(self.msprobe_input_kwargs, {}) + self.assertEqual(ret, 1) + + del self.msprobe_hook + del self.msprobe_input_kwargs + + def test_set_and_get_calls_number(self): + CellProcessor.cell_count = {} + count = self.processor.set_and_get_calls_number("cell") self.assertEqual(count, 0) - self.assertEqual(CellProcessor.cell_count["cell1"], 0) + self.assertEqual(CellProcessor.cell_count["cell"], 0) - def test_set_cell_count_existing_cell(self): - self.processor.set_cell_count("cell1") - count = self.processor.set_cell_count("cell1") + count = self.processor.set_and_get_calls_number("cell") self.assertEqual(count, 1) - self.assertEqual(CellProcessor.cell_count["cell1"], 1) + self.assertEqual(CellProcessor.cell_count["cell"], 1) + + CellProcessor.cell_count = {} def test_reset_cell_stats(self): - self.processor.set_cell_count("cell1") + CellProcessor.cell_count['cell'] = 0 + CellProcessor.cell_stack.append('cell') + CellProcessor.api_parent_node = 'cell' + CellProcessor.module_node['cell'] = 'null' + CellProcessor.cell_bw_hook_kernels['cell'] = 'bw' + CellProcessor.cell_backward_pre_hook.append('backward_pre_hook') + CellProcessor.cell_backward_hook.append('backward_hook') + CellProcessor.reset_cell_stats() self.assertEqual(CellProcessor.cell_count, {}) self.assertEqual(CellProcessor.cell_stack, []) - self.assertEqual(CellProcessor.api_parent_node, "") + self.assertIsNone(CellProcessor.api_parent_node) self.assertEqual(CellProcessor.module_node, {}) + self.assertEqual(CellProcessor.cell_bw_hook_kernels, {}) + self.assertEqual(CellProcessor.cell_backward_pre_hook, []) + self.assertEqual(CellProcessor.cell_backward_hook, []) - @patch('msprobe.core.common.const.Const') - def test_node_hook_begin(self, mock_const): - mock_const.SEP = "." # 确保 SEPARATOR 设置为字符串 - mock_const.START = "start" - cell = MockCell() - self.processor.node_hook("prefix", "start")(cell, "input") - - expected_name = "prefix" + mock_const.SEP + "0" - self.assertEqual(cell.mindstudio_reserved_name, expected_name) - self.assertIn(expected_name, CellProcessor.cell_stack) - self.assertEqual(CellProcessor.api_parent_node, expected_name) - self.scope.begin_module.assert_called_once_with(expected_name) - - @patch('msprobe.core.common.const.Const') - def test_node_hook_end(self, mock_const): - mock_const.START = "start" - cell = MockCell() - self.processor.node_hook("prefix", "start")(cell, "input") - self.processor.node_hook("prefix", "stop")(cell, "input", "output") - - self.assertEqual(len(CellProcessor.cell_stack), 0) - self.assertIsNone(CellProcessor.api_parent_node) - self.scope.end_module.assert_called_once_with(cell.mindstudio_reserved_name) + def test_register_cell_hook(self): + with self.assertRaises(MsprobeException) as context: + self.processor.register_cell_hook([], None, 'config') + self.assertEqual(str(context.exception), '[msprobe] 无效参数：The model cannot be None, when level is "L0" or "mix"') - @patch('msprobe.core.common.const.Const') - def test_multiple_node_hook_calls(self, mock_const): - mock_const.SEP = "." # 确保 SEPARATOR 设置为字符串 - mock_const.START = "start" - cell = MockCell() + with patch('msprobe.mindspore.cell_processor.is_mindtorch') as mock_is_mindtorch, \ + patch('msprobe.mindspore.cell_processor.get_cells_and_names_with_index') as mock_get_cells_and_names, \ + patch('msprobe.mindspore.cell_processor.CellProcessor.build_cell_hook') as mock_build_cell_hook, \ + patch('msprobe.mindspore.cell_processor.get_cell_construct') as mock_get_cell_construct, \ + patch('msprobe.mindspore.cell_processor.is_graph_mode_cell_dump_allowed') \ + as mock_is_graph_mode_cell_dump_allowed, \ + patch.object(logger, 'info') as mock_logger_info: + mock_cell = MagicMock() + mock_sub_cell = MagicMock() + mock_get_cells_and_names.return_value = ({'-1': [('cell', mock_cell), ('sub_cell', mock_sub_cell)]}, {}) + mock_build_cell_hook.return_value = 'forward_pre_hook' + mock_get_cell_construct.return_value = '_construct' + mock_is_graph_mode_cell_dump_allowed.return_value = False - # First call - self.processor.node_hook("prefix", "start")(cell, "input") - expected_name1 = "prefix" + mock_const.SEP + "0" + mock_is_mindtorch.return_value = False + setattr(MagicMock, '_run_construct', '_run_construct') + self.processor.register_cell_hook(mock_cell, None, 'config') + self.assertTrue(mock_sub_cell.__class__.msprobe_construct) + mock_get_cell_construct.assert_called_with('_run_construct') + self.assertEqual(mock_sub_cell.__class__._run_construct, '_construct') + self.assertTrue(mock_sub_cell.msprobe_hook) + mock_build_cell_hook.assert_called_with('Cell.sub_cell.MagicMock.', None) + mock_cell.assert_not_called() + mock_sub_cell.register_forward_pre_hook.assert_called_with('forward_pre_hook') + mock_sub_cell.register_forward_hook.assert_not_called() + mock_logger_info.assert_called_with('The cell hook function is successfully mounted to the model.') - # Second call - self.processor.node_hook("prefix", "start")(cell, "input") - expected_name2 = "prefix" + mock_const.SEP + "1" + del MagicMock._run_construct + del mock_sub_cell.__class__._run_construct + del mock_sub_cell.__class__.msprobe_construct - self.assertEqual(cell.mindstudio_reserved_name, expected_name2) - self.assertEqual(CellProcessor.api_parent_node, expected_name2) + mock_get_cell_construct.reset_mock() + mock_another_sub_cell = MagicMock() + setattr(mock_another_sub_cell.__class__, 'msprobe_construct', True) + mock_get_cells_and_names.return_value = ( + {'-1': [('cell', mock_cell), ('another_sub_cell', mock_another_sub_cell)]}, + {} + ) + self.processor.register_cell_hook(mock_cell, None, 'config') + mock_get_cell_construct.assert_not_called() + mock_another_sub_cell.register_forward_pre_hook.assert_called_with('forward_pre_hook') + mock_another_sub_cell.register_forward_hook.assert_not_called() - # End first call - self.processor.node_hook("prefix", "stop")(cell, "input", "output") - self.assertEqual(len(CellProcessor.cell_stack), 1) # Still one item in stack - self.assertEqual(CellProcessor.api_parent_node, expected_name1) + del mock_another_sub_cell.__class__.msprobe_construct - # End second call - self.processor.node_hook("prefix", "stop")(cell, "input", "output") - self.assertEqual(len(CellProcessor.cell_stack), 0) # Stack should be empty now - self.assertIsNone(CellProcessor.api_parent_node) + mock_build_cell_hook.reset_mock() + mock_get_cell_construct.reset_mock() + mock_another_sub_cell.reset_mock() + setattr(MagicMock, '_call_impl', '_call_impl') + mock_is_mindtorch.return_value = True + self.processor.register_cell_hook(mock_cell, None, 'config') + self.assertTrue(mock_another_sub_cell.__class__.msprobe_construct) + mock_get_cell_construct.assert_called_with('_call_impl') + mock_build_cell_hook.assert_called_with('Module.another_sub_cell.MagicMock.', None) + mock_cell.assert_not_called() + mock_another_sub_cell.register_forward_pre_hook.assert_called_with('forward_pre_hook') + mock_another_sub_cell.register_forward_hook.assert_not_called() + + del MagicMock._call_impl + del mock_another_sub_cell.__class__._call_impl + del mock_another_sub_cell.__class__.msprobe_construct + + def test_build_cell_hook(self): + CellProcessor.reset_cell_stats() + + cell_name = 'Cell.cell.Cell.' + mock_build_data_hook = MagicMock() + mock_backward_data_hook = MagicMock() + target_grad_output = (Tensor([0.5]),) + mock_backward_data_hook.return_value = target_grad_output + mock_build_data_hook.return_value = (None, None, mock_backward_data_hook, None) + mock_cell = MagicMock() - def test_set_and_get_reserved_name(self): - cell = MockCell() - cell.mindstudio_reserved_name = "mindstudio_reserved_name" + with patch.object(_inner_ops, 'CellBackwardHook') as mock_CellBackwardHook: + forward_pre_hook = self.processor.build_cell_hook(cell_name, mock_build_data_hook) + forward_hook = forward_pre_hook.__closure__[2].cell_contents + + mock_bw = mock_CellBackwardHook.return_value + mock_bw.return_value = (Tensor([0.0]),) + args = (Tensor([1.0]),) + target_args = (Tensor([0.0]),) + full_forward_name = f'{cell_name}{Const.FORWARD}.0' + full_backward_name = f'{cell_name}{Const.BACKWARD}.0' + # call testing function - forward_pre_hook + ret = forward_pre_hook(mock_cell, args) + self.assertIsNone(CellProcessor.module_node[full_forward_name]) + self.assertEqual(CellProcessor.cell_stack, [full_forward_name]) + self.assertEqual(CellProcessor.api_parent_node, full_forward_name) + self.scope.begin_module.assert_called_with(full_forward_name) + mock_build_data_hook.assert_called_with('Module', full_forward_name) + self.assertEqual(len(CellProcessor.cell_backward_hook), 1) + mock_CellBackwardHook.assert_called_with(full_backward_name, mock_cell, + CellProcessor.cell_backward_hook[-1]) + mock_bw.register_backward_hook.assert_called_once() + mock_bw.assert_called_with(*args) + self.assertTrue((ret[0] == target_args[0]).all()) + + backward_hook = CellProcessor.cell_backward_hook[-1][full_backward_name] + grad_input = (Tensor([1.0]),) + grad_output = (Tensor([2.0]),) + # call testing function - backward_hook + ret = backward_hook(mock_cell, grad_input, grad_output) + mock_backward_data_hook.assert_called_with(mock_cell, grad_input, grad_output) + self.assertFalse(mock_cell.has_pre_hook_called) + self.assertEqual(CellProcessor.cell_stack, []) + self.assertIsNone(CellProcessor.api_parent_node) + self.scope.end_module.assert_called_with(full_backward_name) + self.assertTrue((ret[0] == target_grad_output[0]).all()) + + mock_build_data_hook.reset_mock() + args = (Tensor([1], dtype=ms.int32),) + full_forward_name = f'{cell_name}{Const.FORWARD}.1' + # call testing function - forward_pre_hook + ret = forward_pre_hook(mock_cell, args) + self.assertIsNone(CellProcessor.module_node[full_forward_name]) + self.assertEqual(CellProcessor.cell_stack, [full_forward_name]) + self.assertEqual(CellProcessor.api_parent_node, full_forward_name) + self.scope.begin_module.assert_called_with(full_forward_name) + self.assertEqual(len(CellProcessor.cell_backward_hook), 1) + mock_build_data_hook.assert_not_called() + + full_forward_name = f'{cell_name}{Const.FORWARD}.0' + CellProcessor.cell_count = {cell_name: 0} + CellProcessor.cell_stack = [full_forward_name] + CellProcessor.api_parent_node = full_forward_name + CellProcessor.module_node = {full_forward_name: None} + self.scope.reset_mock() + mock_CellBackwardHook.reset_mock() + mock_bw.reset_mock() + target_output = Tensor([0.5]) + args = (Tensor([1.0]),) + output = Tensor([2.0]) + mock_bw.return_value = target_output + mock_backward_data_hook.reset_mock() + mock_forward_data_hook_hook = MagicMock() + mock_forward_data_hook_hook.return_value = output + mock_build_data_hook.return_value = (None, mock_forward_data_hook_hook, mock_backward_data_hook, None) + # call testing function - forward_hook + ret = forward_hook(mock_cell, args, output) + self.assertEqual(CellProcessor.cell_count.get(cell_name), 0) + self.assertEqual(CellProcessor.cell_stack, []) + self.assertIsNone(CellProcessor.api_parent_node) + self.scope.end_module.assert_called_with(full_forward_name) + self.assertEqual(mock_bw.call_count, 2) + self.assertEqual(mock_bw.call_args_list[0][0][0], output) + self.assertEqual(mock_bw.call_args_list[1][0][0], target_output) + self.assertEqual(mock_CellBackwardHook.call_count, 1) + self.assertEqual(len(CellProcessor.cell_backward_pre_hook), 1) + self.assertTrue((ret == target_output).all()) + + backward_pre_hook = CellProcessor.cell_backward_pre_hook[-1][full_backward_name] + mock_backward_data_hook.reset_mock() + grad_output = (Tensor([2.0]),) + # call testing function - backward_pre_hook + ret = backward_pre_hook(mock_cell, grad_output) + self.assertTrue(mock_cell.has_pre_hook_called) + self.scope.begin_module.assert_called_with(full_backward_name) + self.assertEqual(CellProcessor.cell_stack, [full_backward_name]) + self.assertEqual(CellProcessor.api_parent_node, full_backward_name) + self.assertEqual(CellProcessor.module_node, {full_forward_name: None, full_backward_name: None}) + self.scope.begin_module.assert_called_with(full_backward_name) + mock_backward_data_hook.assert_not_called() + self.assertIsNone(ret) + + CellProcessor.cell_count = {cell_name: 0} + CellProcessor.cell_stack = [full_forward_name] + CellProcessor.api_parent_node = full_forward_name + CellProcessor.module_node = {full_forward_name: None} + mock_bw.reset_mock() + args = (Tensor([1.0]),) + output = (Tensor([2.0]),) + mock_forward_data_hook_hook.return_value = output + target_output = (Tensor([0.5]),) + # call testing function - forward_hook + ret = forward_hook(mock_cell, args, output) + self.assertEqual(mock_bw.call_count, 2) + self.assertEqual(mock_bw.call_args_list[0][0][0], *output) + self.assertEqual(mock_bw.call_args_list[1][0][0], mock_bw.return_value) + self.assertTrue((ret[0] == target_output[0]).all()) + + CellProcessor.cell_count = {cell_name: 0} + CellProcessor.cell_stack = [full_forward_name] + CellProcessor.api_parent_node = full_forward_name + CellProcessor.module_node = {full_forward_name: None} + CellProcessor.cell_bw_hook_kernels.clear() + CellProcessor.cell_backward_pre_hook.clear() + mock_bw.reset_mock() + mock_bw.return_value = (Tensor([0.5]),) + output = (Tensor([1.0]), Tensor([2.0])) + mock_forward_data_hook_hook.return_value = output + with self.assertRaises(TypeError) as context: + # call testing function - forward_hook + forward_hook(mock_cell, args, output) + self.assertEqual(str(context.exception), + 'The backward pre hook return value size is 1 not equal to output size 2') + mock_bw.assert_called_with(*output) + + self.scope.reset_mock() + backward_pre_hook = CellProcessor.cell_backward_pre_hook[-1][full_backward_name] + # call testing function - backward_pre_hook + ret = backward_pre_hook(mock_cell, grad_output) + self.assertFalse(mock_cell.has_pre_hook_called) + self.scope.begin_module.assert_called_with(full_backward_name) + mock_backward_data_hook.assert_called_with(mock_cell, (), grad_output) + self.assertEqual(CellProcessor.cell_stack, []) + self.assertIsNone(CellProcessor.api_parent_node) + self.assertEqual(CellProcessor.module_node, {full_forward_name: None, full_backward_name: None}) + self.scope.end_module.assert_called_with(full_backward_name) + self.assertIsNone(ret) + + CellProcessor.reset_cell_stats() + + def test_set_construct_info_in_pre_hook(self): CellProcessor.reset_cell_stats() + self.processor.set_construct_info_in_pre_hook('full_name') + self.assertEqual(CellProcessor.module_node['full_name'], None) + self.assertEqual(CellProcessor.cell_stack, ['full_name']) + self.assertEqual(CellProcessor.api_parent_node, 'full_name') + self.scope.begin_module.assert_called_with('full_name') + + self.scope.begin_module.reset_mock() + self.processor.set_construct_info_in_pre_hook('sub_cell_name') + self.assertEqual(CellProcessor.module_node, {'full_name': None, 'sub_cell_name': 'full_name'}) + self.assertEqual(CellProcessor.cell_stack, ['full_name', 'sub_cell_name']) + self.assertEqual(CellProcessor.api_parent_node, 'sub_cell_name') + self.scope.begin_module.assert_called_with('sub_cell_name') - cell_name = "Cell.net.Net.forward" - ret = self.processor.set_and_get_reserved_name(cell, cell_name) - self.assertEqual(ret, cell_name + Const.SEP + "0") - self.assertEqual(cell.mindstudio_reserved_name, ret) - self.assertEqual(CellProcessor.cell_count[cell_name], 0) - self.assertFalse(hasattr(cell, "has_pre_hook_called")) - - cell.has_pre_hook_called = False - ret = self.processor.set_and_get_reserved_name(cell, cell_name) - self.assertEqual(ret, cell_name + Const.SEP + "1") - self.assertEqual(cell.mindstudio_reserved_name, ret) - self.assertEqual(CellProcessor.cell_count[cell_name], 1) - self.assertFalse(cell.has_pre_hook_called) - - cell.has_pre_hook_called = True - cell.mindstudio_reserved_name = "mindstudio_reserved_name" CellProcessor.reset_cell_stats() - ret = self.processor.set_and_get_reserved_name(cell, cell_name) - self.assertEqual(ret, "mindstudio_reserved_name") - self.assertEqual(cell.mindstudio_reserved_name, ret) - self.assertEqual(CellProcessor.cell_count, {}) - self.assertFalse(cell.has_pre_hook_called) - ret = self.processor.set_and_get_reserved_name(cell, cell_name, is_called_by_pre_hook=True) - self.assertEqual(ret, cell_name + Const.SEP + "0") - self.assertEqual(cell.mindstudio_reserved_name, ret) - self.assertEqual(CellProcessor.cell_count[cell_name], 0) - self.assertTrue(cell.has_pre_hook_called) + def test_set_construct_info_in_hook(self): + CellProcessor.reset_cell_stats() + self.processor.set_construct_info_in_hook('full_name') + self.assertIsNone(CellProcessor.api_parent_node) + self.scope.end_module.assert_called_with('full_name') + + self.scope.end_module.reset_mock() + CellProcessor.cell_stack = ['full_name'] + self.processor.set_construct_info_in_hook('full_name') + self.assertEqual(CellProcessor.cell_stack, []) + self.assertIsNone(CellProcessor.api_parent_node) + self.scope.end_module.assert_called_with('full_name') + + self.scope.end_module.reset_mock() + CellProcessor.cell_stack = ['Cell.0', 'Cell.1'] + self.processor.set_construct_info_in_hook('full_name') + self.assertEqual(CellProcessor.cell_stack, ['Cell.0']) + self.assertEqual(CellProcessor.api_parent_node, 'Cell.0') + self.scope.end_module.assert_called_with('full_name') + CellProcessor.reset_cell_stats() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py index 8f5d207c41923175b6efe4f9dc313896f879fd89..2e252f1c8287042b174591fbd0345da77e2307ed 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,16 +16,19 @@ from unittest import TestCase from unittest.mock import patch +from msprobe.core.common.log import logger from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.const import Const as CoreConst from msprobe.mindspore.common.const import Const from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.dump_tool_factory import DumpToolFactory +from msprobe.mindspore.ms_config import StatisticsConfig class TestDumpToolFactory(TestCase): + @patch.object(logger, "error") @patch("msprobe.mindspore.debugger.debugger_config.create_directory") - def test_create(self, _): + def test_create(self, _, mock_logger_error): json_config = { "task": "statistics", "dump_path": "/absolute_path", @@ -35,7 +38,7 @@ class TestDumpToolFactory(TestCase): } common_config = CommonConfig(json_config) - task_config = BaseConfig(json_config) + task_config = StatisticsConfig(json_config) config = DebuggerConfig(common_config, task_config) config.data_mode = [CoreConst.INPUT, CoreConst.OUTPUT] @@ -55,15 +58,25 @@ class TestDumpToolFactory(TestCase): self.assertEqual(str(context.exception), "Valid level is needed.") config.level = Const.KERNEL - with self.assertRaises(Exception) as context: + with self.assertRaises(ValueError): DumpToolFactory.create(config) - self.assertEqual(str(context.exception), "Data dump is not supported in None mode when dump level is kernel.") + mock_logger_error.assert_called_with("Data dump is not supported in None mode when dump level is kernel.") + mock_logger_error.reset_mock() config.execution_mode = Const.GRAPH_GE_MODE config.level = Const.CELL - with self.assertRaises(Exception) as context: - DumpToolFactory.create(config) - self.assertEqual(str(context.exception), "Data dump is not supported in graph_ge mode when dump level is cell.") + with patch('msprobe.mindspore.dump.dump_tool_factory.is_graph_mode_cell_dump_allowed') as \ + mock_is_cell_dump_allowed: + mock_is_cell_dump_allowed.return_value = True + with self.assertRaises(ValueError): + DumpToolFactory.create(config) + mock_logger_error.assert_called_with("Data dump is not supported in graph_ge mode when dump level is cell.") + mock_logger_error.reset_mock() + + mock_is_cell_dump_allowed.return_value = False + with self.assertRaises(Exception) as context: + DumpToolFactory.create(config) + self.assertEqual(str(context.exception), "Cell dump is not supported in graph mode.") config.execution_mode = Const.GRAPH_KBYK_MODE config.level = Const.KERNEL diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py index 329274b19d862c8c0e50af0fdbd051909e6a60d6..ac353fd8832363ae42872272c8bdeda6e0620d69 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,7 @@ # limitations under the License. import os +import sys from unittest import TestCase from unittest.mock import patch @@ -44,10 +45,26 @@ class TestKernelGraphDump(TestCase): self.assertEqual(dumper.dump_json["common_dump_settings"]["file_format"], "bin") self.assertEqual(dumper.dump_json["common_dump_settings"]["input_output"], 2) + _msprobe_c_existed = True + try: + from msprobe.lib import _msprobe_c + except ImportError: + _msprobe_c_existed = False + with patch("msprobe.mindspore.dump.kernel_graph_dump.create_directory"), \ patch("msprobe.mindspore.dump.kernel_graph_dump.logger.info"), \ patch("msprobe.mindspore.dump.kernel_graph_dump.save_json") as mock_save_json: + if _msprobe_c_existed: + dumper.handle() + mock_save_json.assert_not_called() + + _msprobe_c_path = _msprobe_c.__file__ + _msprobe_c_test_path = _msprobe_c_path.replace('_msprobe_c.so', '_msprobe_c_test.so') + os.rename(_msprobe_c_path, _msprobe_c_test_path) + sys.modules.pop('msprobe.lib') + sys.modules.pop('msprobe.lib._msprobe_c') + os.environ["GRAPH_OP_RUN"] = "1" with self.assertRaises(Exception) as context: dumper.handle() @@ -63,3 +80,5 @@ class TestKernelGraphDump(TestCase): del os.environ["MINDSPORE_DUMP_CONFIG"] if "MS_ACL_DUMP_CFG_PATH" in os.environ: del os.environ["MS_ACL_DUMP_CFG_PATH"] + if _msprobe_c_existed: + os.rename(_msprobe_c_test_path, _msprobe_c_path) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py index b484bc9b7cdceec3b8906600b16b2d4fdc6b1b5e..67118ceaf780bd227ada242f2c1ca4b5a925127e 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,7 @@ # limitations under the License. import os +import sys from unittest import TestCase from unittest.mock import patch @@ -41,11 +42,27 @@ class TestKernelGraphOverflowCheck(TestCase): checker = KernelGraphOverflowCheck(config) self.assertEqual(checker.dump_json["common_dump_settings"]["op_debug_mode"], 2) + _msprobe_c_existed = True + try: + from msprobe.lib import _msprobe_c + except ImportError: + _msprobe_c_existed = False + os.environ["MS_ACL_DUMP_CFG_PATH"] = "path" with patch("msprobe.mindspore.overflow_check.kernel_graph_overflow_check.create_directory"), \ patch("msprobe.mindspore.overflow_check.kernel_graph_overflow_check.logger.info"), \ patch("msprobe.mindspore.overflow_check.kernel_graph_overflow_check.save_json") as mock_save_json: + if _msprobe_c_existed: + checker.handle() + mock_save_json.assert_not_called() + + _msprobe_c_path = _msprobe_c.__file__ + _msprobe_c_test_path = _msprobe_c_path.replace('_msprobe_c.so', '_msprobe_c_test.so') + os.rename(_msprobe_c_path, _msprobe_c_test_path) + sys.modules.pop('msprobe.lib') + sys.modules.pop('msprobe.lib._msprobe_c') + os.environ["GRAPH_OP_RUN"] = "1" with self.assertRaises(Exception) as context: checker.handle() @@ -60,3 +77,5 @@ class TestKernelGraphOverflowCheck(TestCase): if "MINDSPORE_DUMP_CONFIG" in os.environ: del os.environ["MINDSPORE_DUMP_CONFIG"] + if _msprobe_c_existed: + os.rename(_msprobe_c_test_path, _msprobe_c_path) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py index c52ea4de2adef5a3c579c3deceece9d84b89309c..9be887eb4be1be00d68b12ca5181f7371dcf075e 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py @@ -21,6 +21,7 @@ from unittest.mock import patch from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.mindspore.ms_config import StatisticsConfig from msprobe.mindspore.dump.kernel_kbyk_dump import KernelKbykDump @@ -36,7 +37,7 @@ class TestKernelKbykDump(TestCase): } common_config = CommonConfig(json_config) - task_config = BaseConfig(json_config) + task_config = StatisticsConfig(json_config) config = DebuggerConfig(common_config, task_config) dumper = KernelKbykDump(config) self.assertEqual(dumper.dump_json["common_dump_settings"]["iteration"], "0|2") @@ -53,6 +54,138 @@ class TestKernelKbykDump(TestCase): if "MINDSPORE_DUMP_CONFIG" in os.environ: del os.environ["MINDSPORE_DUMP_CONFIG"] + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") + def test_handle_when_async_dump_then_pass(self, _): + json_config = { + "task": "statistics", + "dump_path": "/absolute_path", + "rank": [], + "step": [0, 2], + "level": "L2", + "async_dump": True + } + + common_config = CommonConfig(json_config) + task_config = StatisticsConfig(json_config) + config = DebuggerConfig(common_config, task_config) + dumper = KernelKbykDump(config) + self.assertEqual(dumper.dump_json["e2e_dump_settings"]["enable"], False) + + os.environ["MS_ACL_DUMP_CFG_PATH"] = "path" + with patch("msprobe.mindspore.dump.kernel_kbyk_dump.create_directory"), \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.logger.info") as mock_info, \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.save_json") as mock_save_json: + dumper.handle() + self.assertIn("kernel_kbyk_dump.json", mock_save_json.call_args_list[0][0][0]) + mock_info.assert_called_with("/absolute_path/kernel_kbyk_dump.json has been created.") + + self.assertEqual(os.environ.get("MS_ACL_DUMP_CFG_PATH"), None) + if "MINDSPORE_DUMP_CONFIG" in os.environ: + del os.environ["MINDSPORE_DUMP_CONFIG"] + + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") + def test_handle_when_device_then_pass(self, _): + json_config = { + "task": "statistics", + "dump_path": "/absolute_path", + "rank": [], + "step": [0, 2], + "level": "L2", + "statistics": { + "list": [], + "data_mode": ["all"], + "device": "device", + "summary_mode": "statistics" + } + } + + common_config = CommonConfig(json_config) + task_config = StatisticsConfig(json_config["statistics"]) + config = DebuggerConfig(common_config, task_config) + dumper = KernelKbykDump(config) + self.assertEqual(dumper.dump_json["e2e_dump_settings"]["stat_calc_mode"], "device") + + os.environ["MS_ACL_DUMP_CFG_PATH"] = "path" + with patch("msprobe.mindspore.dump.kernel_kbyk_dump.create_directory"), \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.logger.info") as mock_info, \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.save_json") as mock_save_json: + dumper.handle() + self.assertIn("kernel_kbyk_dump.json", mock_save_json.call_args_list[0][0][0]) + mock_info.assert_called_with("/absolute_path/kernel_kbyk_dump.json has been created.") + + self.assertEqual(os.environ.get("MS_ACL_DUMP_CFG_PATH"), None) + if "MINDSPORE_DUMP_CONFIG" in os.environ: + del os.environ["MINDSPORE_DUMP_CONFIG"] + + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") + def test_handle_when_precision_then_pass(self, _): + json_config = { + "task": "statistics", + "dump_path": "/absolute_path", + "rank": [], + "step": [0, 2], + "level": "L2", + "statistics": { + "list": [], + "data_mode": ["all"], + "precision": "low", + "summary_mode": "statistics" + } + } + + common_config = CommonConfig(json_config) + task_config = StatisticsConfig(json_config["statistics"]) + config = DebuggerConfig(common_config, task_config) + dumper = KernelKbykDump(config) + self.assertEqual(dumper.dump_json["e2e_dump_settings"]["device_stat_precision_mode"], "low") + + os.environ["MS_ACL_DUMP_CFG_PATH"] = "path" + with patch("msprobe.mindspore.dump.kernel_kbyk_dump.create_directory"), \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.logger.info") as mock_info, \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.save_json") as mock_save_json: + dumper.handle() + self.assertIn("kernel_kbyk_dump.json", mock_save_json.call_args_list[0][0][0]) + mock_info.assert_called_with("/absolute_path/kernel_kbyk_dump.json has been created.") + + self.assertEqual(os.environ.get("MS_ACL_DUMP_CFG_PATH"), None) + if "MINDSPORE_DUMP_CONFIG" in os.environ: + del os.environ["MINDSPORE_DUMP_CONFIG"] + + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") + def test_handle_when_default_then_pass(self, _): + json_config = { + "task": "statistics", + "dump_path": "/absolute_path", + "rank": [], + "step": [0, 2], + "level": "L2", + "statistics": { + "list": [], + "data_mode": ["all"], + "summary_mode": "statistics" + } + } + + common_config = CommonConfig(json_config) + task_config = StatisticsConfig(json_config) + config = DebuggerConfig(common_config, task_config) + dumper = KernelKbykDump(config) + self.assertEqual(dumper.dump_json["e2e_dump_settings"]["device_stat_precision_mode"], "high") + self.assertEqual(dumper.dump_json["e2e_dump_settings"]["stat_calc_mode"], "host") + self.assertEqual(dumper.dump_json["e2e_dump_settings"]["enable"], True) + + os.environ["MS_ACL_DUMP_CFG_PATH"] = "path" + with patch("msprobe.mindspore.dump.kernel_kbyk_dump.create_directory"), \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.logger.info") as mock_info, \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.save_json") as mock_save_json: + dumper.handle() + self.assertIn("kernel_kbyk_dump.json", mock_save_json.call_args_list[0][0][0]) + mock_info.assert_called_with("/absolute_path/kernel_kbyk_dump.json has been created.") + + self.assertEqual(os.environ.get("MS_ACL_DUMP_CFG_PATH"), None) + if "MINDSPORE_DUMP_CONFIG" in os.environ: + del os.environ["MINDSPORE_DUMP_CONFIG"] + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_handle_tensor(self, _): json_config = { diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_debug_save.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_debug_save.py index 495eedbf41384f820c2ca054fd73192d1966a8bd..dcc0461adb0e132f1587ad0d6de320af7e53f737 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_debug_save.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_debug_save.py @@ -17,7 +17,8 @@ from unittest.mock import patch import mindspore from msprobe.mindspore import PrecisionDebugger -from msprobe.core.common_config import CommonConfig, BaseConfig +from msprobe.core.common_config import CommonConfig +from msprobe.mindspore.ms_config import StatisticsConfig class TestMindsporeDebuggerSave(TestCase): def setUp(self): @@ -35,7 +36,7 @@ class TestMindsporeDebuggerSave(TestCase): } } common_config = CommonConfig(statistics_task_json) - task_config = BaseConfig(statistics_task_json) + task_config = StatisticsConfig(statistics_task_json) with patch("msprobe.mindspore.debugger.precision_debugger.parse_json_config", return_value=(common_config, task_config)), \ patch("msprobe.mindspore.debugger.precision_debugger.set_register_backward_hook_functions"): self.debugger = PrecisionDebugger() @@ -52,26 +53,27 @@ class TestMindsporeDebuggerSave(TestCase): "framework": "mindspore", "dump_data_dir": None, "data": { - "x_tensor.0": { + "x_tensor.0.debug": { "type": "mindspore.Tensor", "dtype": "Float32", - "shape": (1,), - "Max": 1.0, - "Min": 1.0, - "Mean": 1.0, - "Norm": 1.0 + "shape": (1,) }, - "x_tensor_grad.0": { + "x_tensor_grad.0.debug": { "type": "mindspore.Tensor", "dtype": "Float32", - "shape": (1,), - "Max": 2.0, - "Min": 2.0, - "Mean": 2.0, - "Norm": 2.0 + "shape": (1,) } } } + + grad_fn = mindspore.value_and_grad(forward_func, (0, 1)) grad_fn(x, y) - self.assertEqual(self.debugger.service.data_collector.data_writer.cache_debug, result_json) \ No newline at end of file + + result = self.debugger.service.data_collector.data_writer.cache_debug + # Remove 'tensor_stat_index' from all entries in the data dictionary + for key in result["data"]: + if 'tensor_stat_index' in result["data"][key]: + del result["data"][key]['tensor_stat_index'] + + self.assertEqual(result, result_json) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_service.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_service.py index 96d76c17d3ce50cba87feb5a5f392f179758aad9..205a5824059c2394256dc0a81ae0cf8823829b4a 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_service.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_service.py @@ -19,15 +19,13 @@ from collections import defaultdict from unittest.mock import MagicMock, patch from mindspore import nn, ops +import torch from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common.utils import Const -from msprobe.core.data_dump.api_registry import ApiRegistry from msprobe.core.data_dump.scope import BaseScope from msprobe.mindspore.cell_processor import CellProcessor from msprobe.mindspore.common.log import logger -from msprobe.mindspore.common.utils import register_backward_hook_functions -from msprobe.mindspore.dump.hook_cell.api_register import get_api_register from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell from msprobe.mindspore.dump.jit_dump import JitDump from msprobe.mindspore.service import Service @@ -41,36 +39,88 @@ class TestService(unittest.TestCase): self.config_mock.step = [] self.config_mock.rank = [] self.config_mock.task = Const.TENSOR - self.config_mock.framework = Const.MS_FRAMEWORK self.config_mock.list = [] self.config_mock.scope = [] - self.service = Service(self.config_mock) - self.service.model = MagicMock(spec=nn.Cell) - self.service.data_collector = MagicMock() - self.service.primitive_hook_service = MagicMock() - - def tearDown(self) -> None: - get_api_register().restore_all_api() + with patch('msprobe.mindspore.service.build_data_collector'), \ + patch('msprobe.mindspore.service.CellProcessor'), \ + patch('msprobe.mindspore.service.PrimitiveHookService'), \ + patch('msprobe.mindspore.service.get_api_register'): + self.service = Service(self.config_mock) def test_init(self): - self.assertEqual(self.service.config.level, "L0") - self.assertFalse(self.service.switch) - self.assertFalse(self.service.should_stop_service) - self.assertFalse(self.service.start_call) - self.assertTrue(self.service.first_start) - - def test_check_model_valid_with_valid_cell(self): - model = nn.Cell() - model_list = [model] - self.assertEqual(self.service.check_model_valid(model), model) - self.assertEqual(self.service.check_model_valid(model_list), model_list) - - def test_check_model_valid_with_invalid_type(self): - model = nn.Cell() - with self.assertRaises(MsprobeException): - self.service.check_model_valid("not a cell") - with self.assertRaises(MsprobeException): - self.service.check_model_valid(["not a cell", model]) + with patch('msprobe.mindspore.service.build_data_collector') as mock_build_data_collector, \ + patch('msprobe.mindspore.service.CellProcessor') as mock_CellProcessor, \ + patch('msprobe.mindspore.service.PrimitiveHookService') as mock_PrimitiveHookService, \ + patch('msprobe.mindspore.service.get_api_register') as mock_get_api_register, \ + patch.object(Service, 'register_api_hook') as mock_register_api_hook: + self.service = Service(self.config_mock) + self.assertIsNone(self.service.model) + self.assertEqual(self.service.config.level_ori, Const.LEVEL_L0) + self.assertEqual(self.service.config.dump_path, '/tmp/dump') + self.assertEqual(self.service.config.step, []) + self.assertEqual(self.service.config.rank, []) + self.assertEqual(self.service.config.task, Const.TENSOR) + self.assertEqual(self.service.config.list, []) + self.assertEqual(self.service.config.scope, []) + self.assertEqual(self.service.config.level, Const.LEVEL_L0) + mock_build_data_collector.assert_called_with(self.service.config) + mock_CellProcessor.assert_called_with(mock_build_data_collector.return_value.scope) + mock_PrimitiveHookService.assert_called_with(self.service) + self.assertFalse(self.service.switch) + self.assertFalse(self.service.inner_switch) + self.assertFalse(self.service.primitive_switch) + self.assertEqual(self.service.current_iter, 0) + self.assertEqual(self.service.loop, 0) + self.assertEqual(self.service.init_step, 0) + self.assertTrue(self.service.first_start) + self.assertIsNone(self.service.current_rank) + self.assertIsNone(self.service.dump_iter_dir) + self.assertFalse(self.service.start_call) + self.assertFalse(self.service.should_stop_service) + self.assertEqual(self.service.params_grad_info, {}) + self.assertEqual(self.service.hook_handle_dict, {}) + mock_get_api_register.assert_called_with() + mock_register_api_hook.assert_called_with() + + def test_check_model_valid(self): + with patch('msprobe.mindspore.service.is_mindtorch') as mock_is_mindtorch: + mock_is_mindtorch.return_value = False + model = None + self.assertIsNone(self.service.check_model_valid(model)) + model = 'model' + with self.assertRaises(MsprobeException) as context: + self.service.check_model_valid(model) + self.assertEqual(context.exception.code, MsprobeException.INVALID_PARAM_ERROR) + self.assertIn("The 'model' parameter must be a mindspore.nn.Cell or list[mindspore.nn.Cell] type, " + "currently there is a type.", str(context.exception)) + model = nn.Cell() + self.assertEqual(self.service.check_model_valid(model), model) + models = [model] + self.assertEqual(self.service.check_model_valid(models), models) + models = [model, 'model'] + with self.assertRaises(MsprobeException) as context: + self.service.check_model_valid(models) + self.assertEqual(context.exception.code, MsprobeException.INVALID_PARAM_ERROR) + self.assertIn("The 'model' parameter must be a mindspore.nn.Cell or list[mindspore.nn.Cell] type, " + "currently there is a type.", str(context.exception)) + + mock_is_mindtorch.return_value = True + model = 'model' + with self.assertRaises(MsprobeException) as context: + self.service.check_model_valid(model) + self.assertEqual(context.exception.code, MsprobeException.INVALID_PARAM_ERROR) + self.assertIn("The 'model' parameter must be a torch.nn.Module or list[torch.nn.Module] type, " + "currently there is a type.", str(context.exception)) + model = torch.nn.Module() + self.assertEqual(self.service.check_model_valid(model), model) + models = [model] + self.assertEqual(self.service.check_model_valid(models), models) + models = [model, 'model'] + with self.assertRaises(MsprobeException) as context: + self.service.check_model_valid(models) + self.assertEqual(context.exception.code, MsprobeException.INVALID_PARAM_ERROR) + self.assertIn("The 'model' parameter must be a torch.nn.Module or list[torch.nn.Module] type, " + "currently there is a type.", str(context.exception)) def test_update_primitive_counters(self): self.service.primitive_counters = {} @@ -85,35 +135,59 @@ class TestService(unittest.TestCase): self.service.current_rank = 0 self.service.data_collector.tasks_need_tensor_data = [Const.TENSOR] self.service.data_collector.update_dump_paths = MagicMock() - self.service.create_dirs() expected_calls = [ ("/tmp/dump"), ("/tmp/dump/step1/rank0"), "/tmp/dump/step1/rank0/dump_tensor_data" ] - mock_create_directory.assert_has_calls( - [unittest.mock.call(path) for path in expected_calls], any_order=True) - - args, _ = self.service.data_collector.update_dump_paths.call_args - self.assertEqual(args[0].dump_file_path, "/tmp/dump/step1/rank0/dump.json") - self.assertEqual(args[0].stack_file_path, "/tmp/dump/step1/rank0/stack.json") - self.assertEqual(args[0].construct_file_path, "/tmp/dump/step1/rank0/construct.json") - self.assertEqual(args[0].dump_tensor_data_dir, "/tmp/dump/step1/rank0/dump_tensor_data") - self.service.data_collector.initialize_json_file.assert_called_once_with( - framework=Const.MS_FRAMEWORK - ) - + with patch('msprobe.mindspore.service.is_mindtorch') as mock_is_mindtorch: + mock_is_mindtorch.return_value = False + self.service.create_dirs() + mock_create_directory.assert_has_calls( + [unittest.mock.call(path) for path in expected_calls], any_order=True) + + args, _ = self.service.data_collector.update_dump_paths.call_args + self.assertEqual(args[0].dump_file_path, "/tmp/dump/step1/rank0/dump.json") + self.assertEqual(args[0].stack_file_path, "/tmp/dump/step1/rank0/stack.json") + self.assertEqual(args[0].construct_file_path, "/tmp/dump/step1/rank0/construct.json") + self.assertEqual(args[0].dump_tensor_data_dir, "/tmp/dump/step1/rank0/dump_tensor_data") + self.service.data_collector.initialize_json_file.assert_called_once_with( + framework=Const.MS_FRAMEWORK + ) + + mock_create_directory.reset_mock() + self.service.data_collector.update_dump_paths.reset_mock() + self.service.data_collector.initialize_json_file.reset_mock() + + mock_is_mindtorch.return_value = True + self.service.create_dirs() + mock_create_directory.assert_has_calls( + [unittest.mock.call(path) for path in expected_calls], any_order=True) + + args, _ = self.service.data_collector.update_dump_paths.call_args + self.assertEqual(args[0].dump_file_path, "/tmp/dump/step1/rank0/dump.json") + self.assertEqual(args[0].stack_file_path, "/tmp/dump/step1/rank0/stack.json") + self.assertEqual(args[0].construct_file_path, "/tmp/dump/step1/rank0/construct.json") + self.assertEqual(args[0].dump_tensor_data_dir, "/tmp/dump/step1/rank0/dump_tensor_data") + self.service.data_collector.initialize_json_file.assert_called_once_with( + framework=Const.MT_FRAMEWORK + ) + + @patch.object(Service, 'check_model_valid') @patch.object(Service, 'need_end_service', return_value=False) - def test_start_stop_cycle(self, mock_need_end_service): + def test_start_stop_cycle(self, mock_need_end_service, mock_check_model_valid): self.service.model = nn.Cell() - with patch.object(self.service, 'register_cell_hook') as mock_register_hook: - self.should_stop_service = False - self.service.start(self.service.model) - self.assertTrue(self.service.switch) - self.service.stop() - self.assertFalse(self.service.switch) - mock_register_hook.assert_called_once() - mock_need_end_service.assert_called_once() + mock_check_model_valid.return_value = self.service.model + self.should_stop_service = False + self.service.start(self.service.model) + mock_check_model_valid.assert_called_with(self.service.model, None) + self.assertTrue(self.service.switch) + self.service.stop() + self.assertFalse(self.service.switch) + self.service.cell_processor.register_cell_hook.assert_called_once() + mock_need_end_service.assert_called_once() + + self.service.cell_processor.register_cell_hook.reset_mock() def test_should_execute_hook_return_false(self): cell = MagicMock() @@ -174,17 +248,16 @@ class TestService(unittest.TestCase): @patch.object(Service, 'need_end_service', return_value=False) @patch.object(logger, 'info') - @patch.object(Service, 'register_cell_hook') @patch.object(Service, 'register_primitive_hook') @patch.object(Service, 'create_dirs') @patch('msprobe.mindspore.service.get_rank_if_initialized', return_value=0) def test_start_first_time(self, mock_get_rank, mock_create_dirs, mock_register_primitive_hook, - mock_register_cell_hook, mock_logger, mock_need_end_service): + mock_logger, mock_need_end_service): self.service.first_start = True self.service.should_stop_service = False self.service.start(self.service.model) mock_get_rank.assert_called_once() - mock_register_cell_hook.assert_called_once() + self.service.cell_processor.register_cell_hook.assert_called_once() mock_register_primitive_hook.assert_called_once() mock_need_end_service.assert_called_once() mock_create_dirs.assert_called_once() @@ -193,27 +266,29 @@ class TestService(unittest.TestCase): self.assertTrue(self.service.primitive_switch) mock_logger.assert_called_with(f"Dump data will be saved in {self.service.dump_iter_dir}.") + self.service.cell_processor.register_cell_hook.reset_mock() + @patch.object(Service, 'register_primitive_hook') - @patch.object(Service, 'register_cell_hook') @patch.object(Service, 'need_end_service', return_value=False) @patch.object(JitDump, 'set_config') @patch.object(JitDump, 'set_data_collector') - @patch.object(ApiRegistry, 'register_all_api') - def test_start_with_jit_dump_enabled(self, mock_api_set_hook_func, mock_set_data_collector, - mock_set_config, mock_need_end_service, mock_register_cell_hook, - mock_register_primitive_hook): + def test_start_with_jit_dump_enabled(self, mock_set_data_collector, mock_set_config, + mock_need_end_service, mock_register_primitive_hook): self.service.config.level = Const.LEVEL_MIX self.service.first_start = True self.service.should_stop_service = False self.service.start(self.service.model) mock_set_config.assert_called_with(self.service.config) mock_set_data_collector.assert_called_with(self.service.data_collector) - mock_api_set_hook_func.assert_called_once() + self.service.api_register.register_all_api.assert_called_once() mock_need_end_service.assert_called_once() - mock_register_cell_hook.assert_called_once() + self.service.cell_processor.register_cell_hook.assert_called_once() mock_register_primitive_hook.assert_called_once() self.assertTrue(JitDump.jit_dump_switch) + self.service.api_register.register_all_api.reset_mock() + self.service.cell_processor.register_cell_hook.reset_mock() + def test_step_updates(self): CellProcessor.cell_count = {"test_api": 1} HOOKCell.cell_count = {"test_api": 1} @@ -236,14 +311,13 @@ class TestService(unittest.TestCase): self.service.data_collector.backward_data_collect = MagicMock() mock_cell = MagicMock() - mock_cell.mindstudio_reserved_name = "TestCell" mock_input = (MagicMock(),) mock_output = MagicMock() - _, forward_hook, backward_hook, _ = self.service.build_hook(BaseScope.Module_Type_Module, "TestHook") + _, forward_hook, backward_hook, _ = self.service.build_hook(BaseScope.Module_Type_Module, "TestHook.forward.0") forward_hook(mock_cell, mock_input, mock_output) - self.service.data_collector.update_api_or_module_name.assert_called_with('TestCell') + self.service.data_collector.update_api_or_module_name.assert_called_with('TestHook.forward.0') self.service.data_collector.forward_data_collect.assert_called() self.service.data_collector.reset_mock() @@ -252,50 +326,33 @@ class TestService(unittest.TestCase): mock_grad_output = MagicMock() backward_hook(mock_cell, mock_grad_input, mock_grad_output) - self.service.data_collector.update_api_or_module_name.assert_called_with('TestHookbackward.0') + self.service.data_collector.update_api_or_module_name.assert_called_with('TestHook.backward.0') self.service.data_collector.backward_data_collect.assert_called() def test_register_primitive_hook(self): self.service.config.level = Const.LEVEL_MIX primitive_attr = ops.Add() primitive_name = "primitive_api" + mock_model = MagicMock() cell_mock = MagicMock() cell_mock.primitive_api = primitive_attr primitive_combined_name = primitive_name + Const.SEP + primitive_attr.__class__.__name__ - self.service.model.cells_and_names.return_value = [("cell_name", cell_mock)] - self.service.register_primitive_hook() + self.service.model = mock_model + with patch('msprobe.mindspore.service.get_cells_and_names_with_index') as mock_get_cells_and_names: + mock_get_cells_and_names.return_value = ({'-1': [("cell_name", cell_mock)]}, {}) + self.service.register_primitive_hook() self.assertTrue(hasattr(primitive_attr.__class__, '__call__')) self.assertEqual(self.service.primitive_hook_service.wrap_primitive.call_args[0][1], primitive_combined_name) - @patch.object(ApiRegistry, 'initialize_hook') - @patch.object(ApiRegistry, 'register_all_api') @patch("msprobe.mindspore.service.logger.info") - def test_register_hook_new_with_level_mix(self, mock_logger, mock_api_set_hook_func, mock_initialize_hook): + def test_register_hook_new_with_level_mix(self, mock_logger): self.service.config.level = Const.LEVEL_MIX self.service.register_api_hook() - self.service.register_cell_hook() - mock_logger.assert_called_with(f"The cell {self.service.config.task} hook function " - "is successfully mounted to the model.") - mock_api_set_hook_func.assert_called() - mock_initialize_hook.assert_called() - - @patch.object(CellProcessor, 'node_hook') - def test_register_hook_new_with_level_l0(self, mock_node_hook): - global register_backward_hook_functions - self.service.config.level = Const.LEVEL_L0 - cell_mock = MagicMock() - self.service.model.cells_and_names.return_value = [("cell_name", cell_mock)] - register_backward_hook_functions["pre"] = cell_mock.register_backward_pre_hook - register_backward_hook_functions["full"] = cell_mock.register_backward_hook - self.service.register_cell_hook() - cell_mock.register_forward_hook.assert_called() - cell_mock.register_backward_hook.assert_called() - mock_node_hook.assert_called() - register_backward_hook_functions = {} - - def test_register_hook_new_without_model_raises_exception(self): - self.service.config.level = Const.LEVEL_L0 - self.service.model = None - with self.assertRaises(MsprobeException): - self.service.register_cell_hook() + mock_logger.assert_called_with(f'The api {self.service.config.task} hook function ' + 'is successfully mounted to the model.') + self.service.api_register.initialize_hook.assert_called_once() + self.service.api_register.register_all_api.assert_called_once() + + self.service.api_register.initialize_hook.reset_mock() + self.service.api_register.register_all_api.reset_mock() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py index f46f171aa38585ea801f1fd3a9716bd3876a63a5..31fe6254a4638b941babb74a5d63713f1b3a7120 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py @@ -1,7 +1,6 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -13,19 +12,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + from unittest import TestCase from unittest.mock import patch -from msprobe.mindspore.common.const import Const +from msprobe.core.common.log import logger from msprobe.core.common_config import CommonConfig, BaseConfig +from msprobe.mindspore.common.const import Const from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.overflow_check.overflow_check_tool_factory import OverflowCheckToolFactory class TestOverflowCheckToolFactory(TestCase): + @patch.object(logger, "error") @patch("msprobe.mindspore.debugger.debugger_config.create_directory") - def test_create(self, _): + def test_create(self, _, mock_logger_error): json_config = { "task": "overflow_check", "dump_path": "/absolute_path", @@ -45,11 +46,10 @@ class TestOverflowCheckToolFactory(TestCase): config.execution_mode = Const.GRAPH_GE_MODE config.level = "cell" - with self.assertRaises(Exception) as context: + with self.assertRaises(ValueError): OverflowCheckToolFactory.create(config) - self.assertEqual(str(context.exception), - f"Overflow check is not supported in {config.execution_mode} mode " - f"when level is {config.level}.") + mock_logger_error.assert_called_with(f"Overflow check is not supported in {config.execution_mode} mode " + f"when level is {config.level}.") config.level = "kernel" dumper = OverflowCheckToolFactory.create(config) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py index 79deeee08e13273f08f32be26a375d1d26f5d2f1..c9090438397ed3cc8c0acf6255f471e77c106110 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py @@ -1,8 +1,7 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # @@ -13,96 +12,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" + +from collections import defaultdict +import tempfile import unittest -import mindspore as ms -import numpy as np -import os from unittest.mock import Mock, patch -from mindspore import nn +import numpy as np +import mindspore as ms +from mindspore import Tensor, ops -import tempfile from msprobe.core.common.utils import Const from msprobe.mindspore.service import Service -from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell -from collections import defaultdict from msprobe.mindspore.dump.hook_cell.primitive_hooks import PrimitiveHookService -from mindspore.common.tensor import Tensor - - -class DummyModel(nn.Cell): - def __init__(self): - super(DummyModel, self).__init__() - self.dense = nn.Dense(2, 2) - - def construct(self, x): - return self.dense(x) - - -class TestService(unittest.TestCase): - @patch("msprobe.mindspore.debugger.debugger_config.create_directory") - def setUp(self, _): - json_config = { - "task": "statistics", - "dump_path": "/absolute_path", - "rank": [], - "step": [0, 2], - "level": "L1" - } - - common_config = CommonConfig(json_config) - task_config = BaseConfig(json_config) - config = DebuggerConfig(common_config, task_config) - self.service = Service(config) - self.service.model = Mock() - self.service.data_collector = Mock() - self.service.switch = True # Make sure the switch is on for testing - self.service.primitive_switch = True # Make sure the switch is on for testing - - def test_check_model_valid_none(self): - model = None - self.assertIsNone(self.service.check_model_valid(model)) - - def test_check_model_valid_valid_model(self): - model = DummyModel() - self.assertEqual(self.service.check_model_valid(model), model) - - def test_check_model_valid_invalid_model(self): - model = "invalid_model" - with self.assertRaises(MsprobeException) as context: - self.service.check_model_valid(model) - - def test_update_primitive_counters(self): - primitive_name = "test_primitive" - self.service.primitive_hook_service.update_primitive_counters(primitive_name) - self.assertEqual(self.service.primitive_hook_service.primitive_counters[primitive_name], 0) - self.service.primitive_hook_service.update_primitive_counters(primitive_name) - self.assertEqual(self.service.primitive_hook_service.primitive_counters[primitive_name], 1) - - def test_step_updates_iteration(self): - initial_iter = self.service.loop - self.service.step() - self.assertEqual(self.service.loop, initial_iter + 1) - - @patch.object(HOOKCell, 'cell_count', new_callable=lambda: defaultdict(int)) - def test_step_resets_counters(self, _): - # 假设在 step 调用之前已经有一些 primitive_counters - self.service.primitive_hook_service.primitive_counters["test_primitive"] = 5 - self.service.step() - self.assertEqual(self.service.primitive_hook_service.primitive_counters, {}) - self.assertEqual(HOOKCell.cell_count, defaultdict(int)) - - def test_start_calls_update_iter(self): - # 检查是否在调用 start 时调用了 update_iter - with patch.object(self.service.data_collector, 'update_iter') as mock_update_iter: - initial_iter = self.service.loop - init_step = self.service.init_step - self.service.start() - mock_update_iter.assert_called_once_with(initial_iter + init_step) +from msprobe.mindspore.ms_config import StatisticsConfig class TestPrimitiveHookService(unittest.TestCase): @@ -119,21 +45,16 @@ class TestPrimitiveHookService(unittest.TestCase): } common_config = CommonConfig(json_config) - task_config = BaseConfig(json_config) + task_config = StatisticsConfig(json_config) config = DebuggerConfig(common_config, task_config) - self.service = Service(config) - self.service.model = Mock() - self.service.data_collector = Mock() - self.service.switch = True # Make sure the switch is on for testing - - # 模拟一个 service_instance 和 data_collector - self.mock_service_instance = Service(config) - self.mock_service_instance.switch = True - self.mock_service_instance.data_collector = Mock() - self.mock_service_instance.data_collector.dump_file_path = json_config["dump_path"] - # 初始化 PrimitiveHookService - self.primitive_hook_service = PrimitiveHookService(self.mock_service_instance) + with patch('msprobe.mindspore.service.build_data_collector'), \ + patch('msprobe.mindspore.service.CellProcessor'), \ + patch('msprobe.mindspore.service.PrimitiveHookService'), \ + patch('msprobe.mindspore.service.get_api_register'): + self.mock_service_instance = Service(config) + self.mock_service_instance.switch = True + self.primitive_hook_service = PrimitiveHookService(self.mock_service_instance) def tearDown(self): # 测试结束时删除临时目录 @@ -148,7 +69,6 @@ class TestPrimitiveHookService(unittest.TestCase): # 调用 wrap_primitive 获取包装函数通过闭包显式调用backward_hook hook_primitive_inputs = self.primitive_hook_service.wrap_primitive(None, "example").__closure__[0].cell_contents - wrapped_primitive_call = self.primitive_hook_service.wrap_primitive(None, "example") create_backward_hook = hook_primitive_inputs.__closure__[0].cell_contents @@ -163,7 +83,6 @@ class TestPrimitiveHookService(unittest.TestCase): backward_hook(grad_2) self.assertEqual(len(captured_grads), 6) # 捕获到两个梯度 - print(f"1After first backward_hook call, len(captured_grads): {len(captured_grads)}") # 调用到达阈值，验证数据收集 self.assertTrue(self.mock_service_instance.data_collector.backward_output_data_collect.called) @@ -177,7 +96,6 @@ class TestPrimitiveHookService(unittest.TestCase): # 调用 wrap_primitive 获取包装函数通过闭包显式调用backward_hook hook_primitive_inputs = self.primitive_hook_service.wrap_primitive(None, "example").__closure__[0].cell_contents - wrapped_primitive_call = self.primitive_hook_service.wrap_primitive(None, "example") create_backward_hook = hook_primitive_inputs.__closure__[0].cell_contents @@ -214,14 +132,7 @@ class TestPrimitiveHookService(unittest.TestCase): # 调用 wrap_primitive 获取包装函数通过闭包显式调用backward_hook hook_primitive_inputs = self.primitive_hook_service.wrap_primitive(None, "example").__closure__[0].cell_contents - wrapped_primitive_call = self.primitive_hook_service.wrap_primitive(None, "example") - if wrapped_primitive_call.__closure__: - for i, closure in enumerate(wrapped_primitive_call.__closure__): - print(f"Closure[{i}]:", closure.cell_contents) - - if hook_primitive_inputs.__closure__: - for i, closure in enumerate(hook_primitive_inputs.__closure__): - print(f"2Closure[{i}]:", closure.cell_contents) + create_backward_hook = hook_primitive_inputs.__closure__[0].cell_contents backward_hook = create_backward_hook(captured_grads, num_tensors, updated_primitive_name, hook_type) @@ -235,7 +146,6 @@ class TestPrimitiveHookService(unittest.TestCase): backward_hook(grad_2) self.assertEqual(len(captured_grads), 6) # 捕获到两个梯度 - print(f"After first backward_hook call, len(captured_grads): {len(captured_grads)}") # 调用到达阈值，验证数据收集 self.assertTrue(self.mock_service_instance.data_collector.backward_input_data_collect.called) @@ -282,18 +192,15 @@ class TestPrimitiveHookService(unittest.TestCase): updated_primitive_name = "test_primitive_input" # 调用 hook_primitive_inputs - hooked_inputs = self.primitive_hook_service.wrap_primitive(None, "example").__closure__[0].cell_contents(args, - captured_grads_input, - updated_primitive_name) - - # 验证 hooked_inputs 是否正确添加了 hook - for arg, hooked_arg in zip(args, hooked_inputs): - if isinstance(arg, Tensor): - print(f"Captured hooked_arg after hook: {hooked_arg}") - self.assertTrue(hasattr(hooked_arg, 'grad_fn')) - - # 打印调试信息 - print(f"Captured gradients after hook: {captured_grads_input}") + hook_primitive_inputs = self.primitive_hook_service.wrap_primitive(None, "example").__closure__[0].cell_contents + with patch.object(ops, 'HookBackward') as mock_HookBackward: + target_value = Tensor([1.0]) + mock_hbw = mock_HookBackward.return_value + mock_hbw.return_value = target_value + hooked_inputs = hook_primitive_inputs(args, captured_grads_input, updated_primitive_name) + self.assertEqual(mock_HookBackward.call_count, len(args)) + for hooked_input in hooked_inputs: + self.assertTrue((hooked_input == target_value).all()) def test_hook_primitive_outputs(self): # 模拟前向输出 @@ -302,17 +209,16 @@ class TestPrimitiveHookService(unittest.TestCase): updated_primitive_name = "test_primitive_output" # 调用 hook_primitive_outputs - hook_primitive_outputs = self.primitive_hook_service.wrap_primitive(None, "example").__closure__[ - 1].cell_contents - hooked_outputs = hook_primitive_outputs(out, captured_grads_output, updated_primitive_name) - - # 验证 hooked_outputs 是否正确添加了 hook - for tensor, hooked_tensor in zip(out, hooked_outputs): - if isinstance(tensor, Tensor): - self.assertTrue(hasattr(hooked_tensor, 'grad_fn')) - - # 打印调试信息 - print(f"Captured gradients after output hook: {captured_grads_output}") + hook_primitive_outputs = self.primitive_hook_service.wrap_primitive(None, + "example").__closure__[1].cell_contents + with patch.object(ops, 'HookBackward') as mock_HookBackward: + target_value = Tensor([1.0]) + mock_hbw = mock_HookBackward.return_value + mock_hbw.return_value = target_value + hooked_outputs = hook_primitive_outputs(out, captured_grads_output, updated_primitive_name) + self.assertEqual(mock_HookBackward.call_count, len(out)) + for hooked_output in hooked_outputs: + self.assertTrue((hooked_output == target_value).all()) def test_wrapped_primitive_call_args(self): # 模拟前向输入 @@ -325,19 +231,18 @@ class TestPrimitiveHookService(unittest.TestCase): # 调用 wrapped_primitive_call 并检查 hooked_inputs 是否与原始 args 相同 try: - hooked_inputs = wrapped_primitive_call.__closure__[0].cell_contents(args, captured_grads_input, - updated_primitive_name) - for arg, hooked_arg in zip(args, hooked_inputs): - if isinstance(arg, Tensor): - self.assertTrue(hasattr(hooked_arg, 'grad_fn')) - self.assertTrue(np.array_equal(arg.asnumpy(), hooked_arg.asnumpy())) - print(f"Arg type: {type(arg)}, Hooked input type: {type(hooked_arg)}") - else: - self.assertEqual(arg, hooked_arg) + with patch.object(ops, 'HookBackward') as mock_HookBackward: + target_value = Tensor([1.0]) + mock_hbw = mock_HookBackward.return_value + mock_hbw.return_value = target_value + hooked_inputs = wrapped_primitive_call.__closure__[0].cell_contents(args, captured_grads_input, + updated_primitive_name) + self.assertEqual(mock_HookBackward.call_count, len(args)) + for hooked_input in hooked_inputs: + self.assertTrue((hooked_input == target_value).all()) except Exception as e: self.fail(f"wrapped_primitive_call raised an exception: {e}") - def test_update_primitive_counters_multiple(self): # 测试更新 primitive 计数器的功能，增加多个不同名称的测试 primitive_names = ["MatMul", "Conv2D", "ReLU", "Softmax"] @@ -416,13 +321,11 @@ class TestPrimitiveHookService(unittest.TestCase): for captured_grads in captured_grads_sets: updated_primitive_name = "MatMul.Backward" - num_tensors = len(captured_grads) hook = self.primitive_hook_service.wrap_primitive(Mock(), "MatMul") backward_hook = hook(Mock(), captured_grads, updated_primitive_name, Const.INPUT) self.assertIsNotNone(backward_hook) - @patch('msprobe.mindspore.dump.hook_cell.primitive_hooks.ops.HookBackward') def test_wrap_primitive_forward_and_backward_hooks(self, mock_hook_backward): # 模拟前向和后向钩子在同一个 primitive 中的行为 @@ -447,9 +350,6 @@ class TestPrimitiveHookService(unittest.TestCase): self.primitive_hook_service.update_primitive_counters(name) self.assertEqual(self.primitive_hook_service.primitive_counters[name], i) - - - def test_update_primitive_counters(self): primitive_name = "MatMul" self.primitive_hook_service.update_primitive_counters(primitive_name) @@ -496,7 +396,7 @@ class TestPrimitiveHookService(unittest.TestCase): wrapped_func = self.primitive_hook_service.wrap_primitive(mock_origin_func, "MatMul") # 模拟反向传播过程，调用包装的 primitive - with patch.object(self.mock_service_instance.data_collector, 'backward_data_collect') as mock_backward_collect: + with patch.object(self.mock_service_instance.data_collector, 'backward_data_collect'): result = wrapped_func(Mock(), input_tensor) # 验证结果是 Tensor 实例 @@ -544,7 +444,6 @@ class TestPrimitiveHookService(unittest.TestCase): # 测试 create_backward_hook 的功能 captured_grads = [] updated_primitive_name = "MatMul.Backward" - num_tensors = 2 # 创建 backward hook backward_hook = self.primitive_hook_service.wrap_primitive(Mock(), "MatMul") diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py index 0a88476d600958b26eaf6ca20a9a70d35b4221cc..952a6dffbc85eea9dd2db87fa081bdf4bb3cae2a 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py @@ -322,7 +322,7 @@ class TestDataGenerateMethods(unittest.TestCase): low_info = [1, float('-inf')] high_info = [2, float('-inf')] tensor = gen_common_tensor(low_info, high_info, shape, data_dtype, None) - self.assertTrue(torch.allclose(tensor.max(), torch.tensor(2.0), atol = 0.3)) + self.assertTrue(torch.allclose(tensor.max(), torch.tensor(2.0), atol = 0.5)) self.assertTrue(tensor.min() == float('-inf')) low_info = [1, float('nan')] diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_pt_accuracy_server.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_pt_accuracy_server.py index b60cfdc323bed57e1cda1fc2d9db3197638cee4c..726714b7993081044a2ca6909db357d3995ad296 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_pt_accuracy_server.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_pt_accuracy_server.py @@ -86,8 +86,8 @@ class TestServerProtocol(unittest.TestCase): ]) self.server_protocol.transport.write.called_once_with(expected_value) - @patch("msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server.hashlib.md5") - def test_post_process_error(self, mock_hashlib_md5): + @patch("msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server.zlib.crc32") + def test_post_process_error(self, mock_zlib_crc32): self.shared_queue.maxsize = 1 self.server_protocol.send_ack = MagicMock() @@ -99,17 +99,18 @@ class TestServerProtocol(unittest.TestCase): self.server_protocol.send_ack.side_effect = [mock_send_ack_method1, mock_send_ack_method2] self.server_protocol.check_sum = True - mock_hashlib_md5.hexdiges.return_value = "123" + mock_zlib_crc32.return_value = 123 self.server_protocol.rank = 0 self.server_protocol.step = 0 self.server_protocol.post_process() - mock_hashlib_md5.assert_called() + mock_zlib_crc32.assert_called() self.server_protocol.send_ack.assert_any_call(self.server_protocol.ACK_ERROR) self.assertEqual(self.server_protocol.rank, -1) self.assertEqual(self.server_protocol.step, -1) - @patch("msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server.hashlib.md5") - def test_post_process_success(self, _): + @patch("msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server.zlib.crc32") + def test_post_process_success(self, mock_zlib_crc32): + mock_zlib_crc32.return_value = 123 self.shared_queue.maxsize = 1 self.server_protocol.send_ack = MagicMock() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py deleted file mode 100644 index ac28e994e9c8e77f8ae675fec3322eaf64a64321..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py +++ /dev/null @@ -1,20 +0,0 @@ -# coding=utf-8 -import unittest -from msprobe.pytorch.compare import match - - -class TestMatch(unittest.TestCase): - def test_graph_mapping(self): - op1 = "Aten_convolution_1_forward_0.input.0" - op2 = "Torch_conv2d_0_forward_0.input.0" - op3 = "Torch_batch_norm_0_forward_0.input.0" - op4 = "Aten_convolution.default_1_forward_0.input.0" - op5 = "Aten_foo_1_forward_0.input.0" - self.assertTrue(match.graph_mapping.match(op1, op2)) - self.assertTrue(match.graph_mapping.match(op2, op1)) - self.assertTrue(match.graph_mapping.match(op4, op2)) - self.assertTrue(match.graph_mapping.match(op2, op4)) - self.assertFalse(match.graph_mapping.match(op1, op3)) - self.assertFalse(match.graph_mapping.match(op3, op1)) - self.assertFalse(match.graph_mapping.match(op5, op2)) - self.assertFalse(match.graph_mapping.match(op2, op5)) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py index b079e646c4a8f4098bb233e3e6259ef3ebea9c94..e4c8b722b182b8c0a4e82ba1b0eeb1a6ed847ee2 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py @@ -3,16 +3,12 @@ import os import shutil import unittest -import numpy as np import torch -from msprobe.core.common.const import Const from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import ModeConfig -from msprobe.pytorch.compare.pt_compare import PTComparator, compare +from msprobe.pytorch.compare.pt_compare import compare from msprobe.test.core_ut.compare.test_acc_compare import generate_dump_json, generate_stack_json - base_dir1 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_pt_compare1') base_dir2 = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_pt_compare2') @@ -40,36 +36,6 @@ class TestUtilsMethods(unittest.TestCase): if os.path.exists(base_dir2): shutil.rmtree(base_dir2) - def test_read_npy_data_bf16(self): - generate_bf16_pt(base_dir1) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - pt_comparator = PTComparator(mode_config) - result = pt_comparator.read_npy_data(base_dir1, 'bf16.pt') - - target_result = torch.tensor([1, 2, 3, 4], dtype=torch.float32).numpy() - self.assertTrue(np.array_equal(result, target_result)) - - def test_read_npy_data_dict(self): - generate_dict_pt(base_dir1) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - pt_comparator = PTComparator(mode_config) - - with self.assertRaises(CompareException) as context: - result = pt_comparator.read_npy_data(base_dir1, 'dict.pt') - self.assertEqual(context.exception.code, CompareException.DETACH_ERROR) - def test_compare(self): generate_dump_json(base_dir2) generate_stack_json(base_dir2) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..558df47a108f27858cc571f6854ca3f403fc6fee --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare_utils.py @@ -0,0 +1,72 @@ +import os +import shutil +import threading +import unittest +from unittest import mock +from unittest.mock import patch + +import numpy as np + +from msprobe.pytorch.compare import utils +from msprobe.pytorch.compare.utils import read_pt_data +from msprobe.test.core_ut.compare.test_acc_compare import generate_pt +from msprobe.core.common.utils import CompareException + + +base_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), f'test_pt_compare_utils_data') +pt_dir = os.path.join(base_dir, f'dump_data_dir') + + +class TestReadPtData(unittest.TestCase): + + def setUp(self): + os.makedirs(base_dir, mode=0o750, exist_ok=True) + os.makedirs(pt_dir, mode=0o750, exist_ok=True) + + self.lock = threading.Lock() + + def tearDown(self): + if os.path.exists(pt_dir): + shutil.rmtree(pt_dir) + if os.path.exists(base_dir): + shutil.rmtree(base_dir) + + def test_read_pt_data_normal(self): + generate_pt(pt_dir) + result = read_pt_data(pt_dir, 'Functional.linear.0.forward.input.0.pt') + expected = np.array([1.0, 2.0, 3.0, 4.0]) + self.assertTrue(np.array_equal(result, expected)) + + def test_read_pt_data_no_file_name(self): + result = read_pt_data(pt_dir, None) + self.assertEqual(result, None) + + @patch.object(utils, 'load_pt') + @patch.object(utils, 'FileChecker') + def test_read_pt_data_runtime_error(self, mock_file_checker_class, mock_load_pt): + mock_file_checker = mock.Mock() + mock_file_checker.common_check.return_value = 'fake/path/file.pt' + mock_file_checker_class.return_value = mock_file_checker + + mock_load_pt.side_effect = RuntimeError('failed to load') + + with self.assertRaises(CompareException) as context: + read_pt_data('fake/path', 'file.pt') + self.assertEqual(context.exception.code, CompareException.INVALID_FILE_ERROR) + + @patch.object(utils, 'load_pt') + @patch.object(utils, 'FileChecker') + def test_read_pt_data_attribute_error(self, mock_file_checker_class, mock_load_pt): + mock_file_checker = mock.Mock() + mock_file_checker.common_check.return_value = 'fake/path/file.pt' + mock_file_checker_class.return_value = mock_file_checker + + class FakeTensor: + def detach(self): + raise AttributeError('no detach') + + mock_load_pt.return_value = FakeTensor() + + with self.assertRaises(CompareException) as context: + read_pt_data('fake/path', 'file.pt') + self.assertEqual(context.exception.code, CompareException.DETACH_ERROR) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger/test_pt_debugger_config.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger/test_pt_debugger_config.py index 4fc27c267ebe65ea46ecf0f17bc47ff702eb241d..3bc7ad866a9012e6797f94729db0be04192fc376 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger/test_pt_debugger_config.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger/test_pt_debugger_config.py @@ -1,6 +1,7 @@ import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch +import torch from msprobe.core.common.const import Const from msprobe.core.common.exceptions import MsprobeException from msprobe.pytorch.debugger.debugger_config import DebuggerConfig @@ -46,28 +47,95 @@ class TestDebuggerConfig(unittest.TestCase): self.assertEqual(debugger.nfs_path, "./nfs_path") self.assertEqual(debugger.port, 8080) - def test_valid_task_and_level(self): - config = DebuggerConfig(self.common_config, self.task_config, "tensor", None, "L1") - config.check_kwargs() + def test_check_kwargs_with_invalid_task(self): + self.common_config.task = "invalid_task" + with self.assertRaises(MsprobeException) as context: + DebuggerConfig(self.common_config, self.task_config, None, None, None) + self.assertIn(f"The task is not in the {Const.TASK_LIST}", str(context.exception)) - def test_invalid_task(self): + def test_check_kwargs_with_invalid_level(self): + self.common_config.level = "invalid_level" with self.assertRaises(MsprobeException) as context: - config = DebuggerConfig(self.common_config, self.task_config, "invalid_task", None, "L1") - config.check_kwargs() - self.assertIn("not in the", str(context.exception)) + DebuggerConfig(self.common_config, self.task_config, None, None, None) + self.assertIn(f"The level is not in the {Const.LEVEL_LIST}.", str(context.exception)) - def test_invalid_level(self): + def test_check_kwargs_with_invalid_dump_path(self): + self.common_config.dump_path = None with self.assertRaises(MsprobeException) as context: - config = DebuggerConfig(self.common_config, self.task_config, "tensor", None, "invalid_level") - config.check_kwargs() - self.assertIn("not in the", str(context.exception)) + DebuggerConfig(self.common_config, self.task_config, None, None, None) + self.assertIn(f"The dump_path not found.", str(context.exception)) - def test_missing_dump_path(self): + def test_check_kwargs_with_invalid_async_dump(self): + self.common_config.async_dump = 1 with self.assertRaises(MsprobeException) as context: - self.common_config.dump_path = None - config = DebuggerConfig(self.common_config, self.task_config, "tensor", None, "L1") - config.check_kwargs() - self.assertIn("dump_path not found", str(context.exception)) + DebuggerConfig(self.common_config, self.task_config, None, None, None) + self.assertIn(f"The parameters async_dump should be bool.", str(context.exception)) + + def test_check_kwargs_with_async_dump_and_debug(self): + self.common_config.async_dump = True + self.common_config.task = Const.TENSOR + self.common_config.level = Const.LEVEL_DEBUG + self.task_config.list = ["linear"] + config = DebuggerConfig(self.common_config, self.task_config, None, None, None) + self.assertEqual(config.list, []) + + def test_check_kwargs_with_async_dump_and_not_debug(self): + self.common_config.async_dump = True + self.common_config.task = Const.TENSOR + self.common_config.level = Const.LEVEL_MIX + self.task_config.list = [] + with self.assertRaises(MsprobeException) as context: + DebuggerConfig(self.common_config, self.task_config, None, None, None) + self.assertIn(f"the parameters list cannot be empty.", str(context.exception)) + + def test_check_kwargs_with_structure_task(self): + self.common_config.task = Const.STRUCTURE + self.common_config.level = Const.LEVEL_L1 + config = DebuggerConfig(self.common_config, self.task_config, None, None, None) + self.assertEqual(config.level, Const.LEVEL_MIX) + + @patch('msprobe.pytorch.debugger.debugger_config.logger') + def test_check_model_with_l1(self, mock_logger): + config = DebuggerConfig(self.common_config, self.task_config, None, None, None) + instance = MagicMock() + instance.model = MagicMock() + config.check_model(instance, None, None) + mock_logger.info_on_rank_0.assert_called_once_with( + "The current level is not L0 or mix level and token_range is None, so the model parameter will not be used" + ) + + def test_check_model_with_model_is_none(self): + self.common_config.level = Const.LEVEL_L0 + instance = MagicMock() + instance.model = None + config = DebuggerConfig(self.common_config, self.task_config, None, None, None) + with self.assertRaises(MsprobeException) as context: + config.check_model(instance, None, None) + self.assertIn("missing the parameter 'model'", str(context.exception)) + + def test_check_model_with_single_model(self): + self.common_config.level = Const.LEVEL_MIX + model1 = torch.nn.ReLU() + model2 = torch.nn.Linear(2, 2) + + instance = MagicMock() + instance.model = model1 + config = DebuggerConfig(self.common_config, self.task_config, None, None, None) + config.check_model(instance, model2, None) + + self.assertEqual(instance.model, model2) + + def test_check_model_with_incorrect_model(self): + self.common_config.level = Const.LEVEL_L0 + model1 = torch.nn.ReLU() + model2 = [torch.nn.Linear(2, 2), torch.nn.ReLU(), "test_model"] + + instance = MagicMock() + instance.model = model1 + config = DebuggerConfig(self.common_config, self.task_config, None, None, None) + with self.assertRaises(MsprobeException) as context: + config.check_model(instance, model2, None) + self.assertIn("must be a torch.nn.Module or list[torch.nn.Module]", str(context.exception)) def test_check_and_adjust_config_with_l2_scope_not_empty(self): self.common_config.dump_path = "./dump_path" @@ -100,3 +168,50 @@ class TestDebuggerConfig(unittest.TestCase): debugger = DebuggerConfig(self.common_config, self.task_config, None, None, None) debugger._check_and_adjust_config_with_l2() self.assertIn("Functional.conv2d.0.forward", self.task_config.list) + + def test_check_and_adjust_config_with_l2_task_not_tensor(self): + self.common_config.dump_path = "./dump_path" + self.common_config.task = Const.STATISTICS + + self.task_config.scope = [] + self.task_config.list = ["Functional.conv2d.0.forward"] + debugger = DebuggerConfig(self.common_config, self.task_config, None, None, None) + with self.assertRaises(MsprobeException) as context: + debugger._check_and_adjust_config_with_l2() + self.assertIn("the task must be set to tensor", str(context.exception)) + + def test_check_statistics_config_task_not_statistics(self): + self.common_config.dump_path = "./dump_path" + self.common_config.task = Const.TENSOR + + debugger = DebuggerConfig(self.common_config, self.task_config, None, None, None) + debugger._check_statistics_config(self.task_config) + self.assertFalse(hasattr(debugger, "tensor_list")) + + def test_check_statistics_config_not_tensor_list(self): + self.common_config.dump_path = "./dump_path" + self.common_config.task = Const.STATISTICS + delattr(self.task_config, "tensor_list") + + debugger = DebuggerConfig(self.common_config, self.task_config, None, None, None) + debugger._check_statistics_config(self.task_config) + self.assertEqual(debugger.tensor_list, []) + + def test_check_statistics_config_debug_level(self): + self.common_config.dump_path = "./dump_path" + self.common_config.task = Const.STATISTICS + self.common_config.level = Const.DEBUG + + debugger = DebuggerConfig(self.common_config, self.task_config, None, None, None) + self.task_config.tensor_list = ["Functional.conv2d"] + debugger._check_statistics_config(self.task_config) + self.assertEqual(debugger.tensor_list, []) + + def test_check_statistics_config_success(self): + self.common_config.dump_path = "./dump_path" + self.common_config.task = Const.STATISTICS + + self.task_config.tensor_list = ["Functional.conv2d"] + debugger = DebuggerConfig(self.common_config, self.task_config, None, None, None) + debugger._check_statistics_config(self.task_config) + self.assertEqual(debugger.tensor_list, self.task_config.tensor_list) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger/test_pt_debugger_start.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger/test_pt_debugger_start.py new file mode 100644 index 0000000000000000000000000000000000000000..9ef667de48864d851181176eb052cb92042dc7c6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger/test_pt_debugger_start.py @@ -0,0 +1,89 @@ +import os +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import TensorDataset, DataLoader +import unittest +from msprobe.pytorch.debugger.precision_debugger import PrecisionDebugger +from msprobe.core.common.file_utils import load_json +import shutil + +# 生成随机分类数据 +X = torch.randn(100, 2) +y = ((X[:, 0] + X[:, 1]) > 0).float().reshape(-1, 1) + +# 创建数据加载器 +dataset = TensorDataset(X, y) +dataloader = DataLoader(dataset, batch_size=10) + +# 定义单层神经网络 +class SingleLayerNet(nn.Module): + def __init__(self): + super().__init__() + self.layer = nn.Linear(2, 1) + self.sigmoid = nn.Sigmoid() + + def forward(self, x): + return self.sigmoid(x) + + +class MultiStartDebugger: + debugger = None + dump_path = None + hooked_model = [] + + @classmethod + def init(cls, dump_path): + cls.dump_path = dump_path + cls.debugger = PrecisionDebugger(task="statistics", level="L0", dump_path=dump_path) + + @classmethod + def debugger_start(cls, model, tag): + cls.debugger.service.first_start = True if model not in cls.hooked_model else False + cls.debugger.service.config.dump_path = os.path.join(cls.dump_path, tag) + cls.debugger.start(model=model) + if model not in cls.hooked_model: + cls.hooked_model.append(model) + + @classmethod + def debugger_stop(cls): + cls.debugger.stop() + cls.debugger.service.reset_status() + + @classmethod + def debugger_step(cls): + cls.debugger.step() + + +class TestPTDebuggerStart(unittest.TestCase): + def test_debugger_multiple_start(self): + dump_path = "./test_debugger_multiple_start_dump" + + model1 = SingleLayerNet() + model2 = SingleLayerNet() + MultiStartDebugger.init(dump_path) + + for batch_X, batch_y in dataloader: + MultiStartDebugger.debugger_start(model=model1, tag="model1") + output1 = model1(batch_X) + MultiStartDebugger.debugger_stop() + + MultiStartDebugger.debugger_start(model=model2, tag="model2") + output2 = model2(batch_X) + MultiStartDebugger.debugger_stop() + MultiStartDebugger.debugger_step() + + model1_dump_path = os.path.join(dump_path, "model1") + self.assertTrue(os.path.exists(model1_dump_path)) + self.assertEqual(len(os.listdir(model1_dump_path)), 10) + model1_construct_json = load_json(os.path.join(model1_dump_path, "step0", "rank", "construct.json")) + self.assertEqual(len(model1_construct_json), 1) + + model2_dump_path = os.path.join(dump_path, "model2") + self.assertTrue(os.path.exists(model2_dump_path)) + self.assertEqual(len(os.listdir(model2_dump_path)), 10) + model2_construct_json = load_json(os.path.join(model2_dump_path, "step0", "rank", "construct.json")) + self.assertEqual(len(model2_construct_json), 1) + + shutil.rmtree(dump_path) + diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger_save/test_debugger_save_pytorch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger_save/test_debugger_save_pytorch.py new file mode 100644 index 0000000000000000000000000000000000000000..8d2f4f3fbcf571e7a61fcef1f6a01118cb2db6df --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/debugger_save/test_debugger_save_pytorch.py @@ -0,0 +1,449 @@ +import unittest +import os +import json +import torch +import numpy as np +import shutil + +from msprobe.pytorch import PrecisionDebugger + +current_file = __file__ +parent_dir = os.path.abspath(os.path.dirname(current_file)) +test_dir = os.path.join(parent_dir, "test_dir") + +def deep_compare(obj1, obj2, float_tolerance=1e-5): + """ + Recursively compare two objects to check if they are the same. + Supports nested dictionaries and lists. + """ + if type(obj1) != type(obj2): + return False + if isinstance(obj1, dict): + if obj1.keys() != obj2.keys(): + return False + return all(deep_compare(obj1[key], obj2[key]) for key in obj1) + if isinstance(obj1, (tuple, list)): + if len(obj1) != len(obj2): + return False + return all(deep_compare(item1, item2) for item1, item2 in zip(obj1, obj2)) + if isinstance(obj1, (int, float)): + return abs(obj1 - obj2) < float_tolerance + return obj1 == obj2 + +class TestDebuggerSave(unittest.TestCase): + @staticmethod + def write_config_json(step, async_dump, mode, dump_path, config_file_path): + task = "tensor" if mode == "tensor" else "statistics" + statistics_summary_mode = "statistics" if mode == "statistics" else "md5" + config = { + "task": task, + "dump_path": dump_path, + "rank": [], + "step": step, + "level": "debug", + "enable_dataloader": False, + "async_dump": async_dump, + "statistics": { + "summary_mode": statistics_summary_mode, + } + } + with open(config_file_path, "w", encoding="utf-8") as f: + json.dump(config, f, indent=4, ensure_ascii=False) + + @staticmethod + def read_debug_json_into_dict(debug_json_path): + with open(debug_json_path, "r", encoding="utf-8") as f: + debug_json = json.load(f) + return debug_json + + + @staticmethod + def check_real_pt(pt_path, target_pt_tensor, check_values=True, rtol=1e-5, atol=1e-8): + """ + Enhanced version with optional value comparison. + + Args: + pt_path (str): Path to the .pt file + target_pt_tensor: Target torch tensor to compare + check_values (bool): If True, also compare array values + rtol, atol: Relative and absolute tolerances for value comparison + + Returns: + bool: True if all checks pass + """ + # Load the pt file + try: + pt_data = torch.load(pt_path) + except FileNotFoundError: + print(f"Error: The file {pt_path} does not exist.") + return False + except Exception as e: + print(f"Error loading pt file: {e}") + return False + # Check shapes + if pt_data.shape != target_pt_tensor.shape: + print(f"Shape mismatch: pt data shape is {pt_data.shape}, target tensor shape is {target_pt_tensor.shape}") + return False + # Check dtypes + if pt_data.dtype != target_pt_tensor.dtype: + print(f"Shape mismatch: pt data dtype is {pt_data.dtype}, target tensor dtype is {target_pt_tensor.dtype}") + return False + # Optionally check values + if check_values: + if not torch.allclose(pt_data, target_pt_tensor, rtol=rtol, atol=atol): + print("Value mismatch: pt data and target tensor values do not match within the specified tolerances.") + return False + return True + + def setUp(self): + if not os.path.exists(test_dir): + os.makedirs(test_dir) + PrecisionDebugger._instance = None + + def tearDown(self): + if os.path.exists(test_dir): + shutil.rmtree(test_dir) + PrecisionDebugger._instance = None + + def test_save_real_tensor(self): + data = {"a": torch.Tensor([1., 2.])} + step = [] + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check pt file + pt_path = os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "data_dict.0.debug.a.pt") + assert self.check_real_pt(pt_path, data["a"]) + # check debug json + target_debug_info = { + "a": { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "requires_grad": False, + "data_name": "data_dict.0.debug.a.pt" + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + def test_save_md5(self): + data = {"a": torch.Tensor([1., 2.])} + step = [] + async_dump = False + mode = "md5" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check debug json + target_debug_info = { + "a": { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "requires_grad": False, + "md5": "2e3fa576" + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + def test_save_multiple_steps(self): + data = {"a": torch.Tensor([1., 2.])} + step = [0, 1, 2] + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + for _ in step: + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check pt file + for i in step: + pt_path = os.path.join(dump_path, f"step{i}", "rank", "dump_tensor_data", "data_dict.0.debug.a.pt") + assert self.check_real_pt(pt_path, data["a"]) + # check debug json + target_debug_info = { + "a": { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "requires_grad": False, + "data_name": "data_dict.0.debug.a.pt" + } + } + for i in step: + debug_json_path = os.path.join(dump_path, f"step{i}", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + def test_async_save_tensor(self): + data = {"a": torch.Tensor([1., 2.])} + step = [] + async_dump = True + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + + # check pt file + pt_path = os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "data_dict.0.debug.a.pt") + assert self.check_real_pt(pt_path, data["a"]) + + # check debug json + target_debug_info = { + "a": { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "data_name": "data_dict.0.debug.a.pt", + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "requires_grad": False, + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + def test_async_save_md5(self): + # async_dump case, md5 configuration not working,only save statistics + data = {"a": torch.Tensor([1., 2.])} + step = [] + async_dump = True + mode = "md5" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + # check debug json + target_debug_info = { + "a": { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "requires_grad": False, + } + } + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"]["data_dict.0.debug"], target_debug_info) + + def test_save_multiple_times(self): + data = {"a": torch.Tensor([1., 2.])} + step = [] + call_times = 3 + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + for _ in range(call_times): + PrecisionDebugger.save(data, "data_dict", save_backward=False) + PrecisionDebugger.step() + + # check pt file + for i in range(call_times): + pt_path = os.path.join(dump_path, "step0", "rank", "dump_tensor_data", f"data_dict.{i}.debug.a.pt") + assert self.check_real_pt(pt_path, data["a"]) + + # check debug json + for i in range(call_times): + target_debug_info = { + "a": { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "requires_grad": False, + "data_name": f"data_dict.{i}.debug.a.pt" + } + } + + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + assert deep_compare(debug_json_dict["data"][f"data_dict.{i}.debug"], target_debug_info) + + def test_save_backward(self): + x = torch.Tensor([1., 2.]) + target_x_grad = torch.Tensor([1., 1.]) + def _forward_simple_func(x): + PrecisionDebugger.save(x, "x_tensor") + return x.sum() + step = [] + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + x.requires_grad = True + loss = _forward_simple_func(x) + loss.backward() + PrecisionDebugger.step() + x_info_list = [ + x, + os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "x_tensor.0.debug.pt"), + "x_tensor.0.debug", + { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "requires_grad": True, + "data_name": "x_tensor.0.debug.pt" + }, + ] + x_grad_info_list = [ + target_x_grad, + os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "x_tensor_grad.0.debug.pt"), + "x_tensor_grad.0.debug", + { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 1.0, + "Min": 1.0, + "Mean": 1.0, + "Norm": 1.4142135381698608, + "requires_grad": False, + "data_name": "x_tensor_grad.0.debug.pt" + }, + ] + check_list = [x_info_list, x_grad_info_list] + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + for check_info in check_list: + target_tensor, target_tensor_path, target_tensor_key, target_tensor_info = check_info + assert self.check_real_pt(target_tensor_path, target_tensor) + assert deep_compare(debug_json_dict["data"][target_tensor_key], target_tensor_info) + + def test_save_compilcated_data_structure_backward(self): + x = torch.Tensor([1., 2.]) + target_x_grad = torch.Tensor([1., 1.]) + def _forward_complicated_func(x): + complicated_structure = [{"a_key": x}] + PrecisionDebugger.save(complicated_structure, "complicated_structure") + return complicated_structure[0]["a_key"].sum() + step = [] + async_dump = False + mode = "tensor" + dump_path = os.path.join(test_dir, "debug_save") + config_file_path = os.path.join(test_dir, "config.json") + self.write_config_json(step, async_dump, mode, dump_path, config_file_path) + debugger = PrecisionDebugger(config_file_path) + x.requires_grad = True + loss = _forward_complicated_func(x) + loss.backward() + PrecisionDebugger.step() + complicated_structure_info_list = [ + x, + os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "complicated_structure.0.debug.0.a_key.pt"), + "complicated_structure.0.debug", + [ + { + "a_key": { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 2.0, + "Min": 1.0, + "Mean": 1.5, + "Norm": 2.2360680103302, + "requires_grad": True, + "data_name": "complicated_structure.0.debug.0.a_key.pt" + } + } + ], + ] + complicated_structure_grad_info_list = [ + target_x_grad, + os.path.join(dump_path, "step0", "rank", "dump_tensor_data", "complicated_structure_grad.0.debug.0.a_key.pt"), + "complicated_structure_grad.0.debug", + [ + { + "a_key": { + "type": "torch.Tensor", + "dtype": "torch.float32", + "shape": [ + 2 + ], + "Max": 1.0, + "Min": 1.0, + "Mean": 1.0, + "Norm": 1.4142135381698608, + "requires_grad": False, + "data_name": "complicated_structure_grad.0.debug.0.a_key.pt" + } + } + ], + ] + check_list = [complicated_structure_info_list, complicated_structure_grad_info_list] + debug_json_path = os.path.join(dump_path, "step0", "rank", "debug.json") + debug_json_dict = self.read_debug_json_into_dict(debug_json_path) + for check_info in check_list: + target_tensor, target_tensor_path, target_tensor_key, target_tensor_info = check_info + assert self.check_real_pt(target_tensor_path, target_tensor) + assert deep_compare(debug_json_dict["data"][target_tensor_key], target_tensor_info) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_module_dump.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_module_dump.py index 5aaf0820a78339ff4f1cc5d28aff8762bae31a39..4ba3556c277f3326520547a6124170f32a9cc8e8 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_module_dump.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_module_dump.py @@ -16,47 +16,68 @@ import unittest from unittest.mock import patch, MagicMock -import torch -import torch.nn as nn +from torch import nn -from msprobe.core.data_dump.api_registry import ApiRegistry -from msprobe.pytorch import PrecisionDebugger -from msprobe.pytorch.hook_module.api_register import get_api_register -from msprobe.pytorch.service import torch_version_above_or_equal_2 +from msprobe.pytorch.common.log import logger +from msprobe.pytorch.dump.module_dump.module_dump import ModuleDumper +from msprobe.pytorch.dump.module_dump.module_processer import ModuleProcesser class TestModuleDumper(unittest.TestCase): - @classmethod - def setUpClass(cls): - PrecisionDebugger._instance = None - get_api_register().restore_all_api() + def setUp(self): + self.service = MagicMock() + with patch('msprobe.pytorch.dump.module_dump.module_dump.get_api_register'): + self.module_dumper = ModuleDumper(self.service) - @classmethod - def tearDownClass(cls): - PrecisionDebugger._instance = None - get_api_register().restore_all_api() + def test__init__(self): + self.service = MagicMock() + with patch('msprobe.pytorch.dump.module_dump.module_dump.get_api_register') as mock_get_api_register: + self.module_dumper = ModuleDumper(self.service) + self.assertEqual(self.module_dumper.service, self.service) + mock_get_api_register.assert_called_once() - def setUp(self): - self.module = nn.Linear(8, 4) - debugger = PrecisionDebugger(dump_path="./") - self.module_dumper = debugger.module_dumper + def test_start_module_dump(self): + module = nn.Module() + with patch.object(logger, 'info_on_rank_0') as mock_info: + module.msprobe_hook = True + ModuleProcesser.enable_module_dump = False + self.module_dumper.api_register.restore_all_api.reset_mock() + self.module_dumper.start_module_dump(module, 'dump_name') + mock_info.assert_called_with('The init dump is enabled, and the module dump function will not be available.') + self.assertFalse(ModuleProcesser.enable_module_dump) + self.module_dumper.api_register.restore_all_api.assert_not_called() + self.assertFalse(hasattr(module, 'msprobe_module_dump')) + + del module.msprobe_hook + mock_info.reset_mock() + self.module_dumper.start_module_dump(module, 'dump_name') + mock_info.assert_not_called() + self.assertTrue(ModuleProcesser.enable_module_dump) + self.module_dumper.api_register.restore_all_api.assert_called_once() + self.module_dumper.service.module_processor.register_module_hook.assert_called_with( + module, + self.module_dumper.service.build_hook, + recursive=False, + module_names=['dump_name'] + ) + self.assertTrue(module.msprobe_module_dump) + ModuleProcesser.enable_module_dump = False + + self.module_dumper.api_register.restore_all_api.reset_mock() + self.module_dumper.service.module_processor.register_module_hook.reset_mock() + self.module_dumper.start_module_dump(module, 'dump_name') + mock_info.assert_not_called() + self.assertTrue(ModuleProcesser.enable_module_dump) + self.module_dumper.api_register.restore_all_api.assert_called_once() + self.module_dumper.service.module_processor.register_module_hook.assert_not_called() + + ModuleProcesser.enable_module_dump = False def test_stop_module_dump(self): - self.module_dumper.hook_handle_list.extend([1, 2, 3]) - with patch.object(ApiRegistry, 'register_all_api') as mock_api_register: - mock_handle1 = MagicMock(spec=torch.utils.hooks.RemovableHandle) - mock_handle2 = MagicMock(spec=torch.utils.hooks.RemovableHandle) - self.module_dumper.hook_handle_list.extend([mock_handle1, mock_handle2]) - - self.module_dumper.stop_module_dump() - mock_handle1.remove.assert_called_once() - mock_handle2.remove.assert_called_once() - self.assertEqual(self.module_dumper.hook_handle_list, []) - mock_api_register.assert_called_once() - - def test_register_hook(self): - self.module_dumper.register_hook(self.module, "TestModule") - if torch_version_above_or_equal_2: - self.assertEqual(len(self.module_dumper.hook_handle_list), 6) - else: - self.assertEqual(len(self.module_dumper.hook_handle_list), 5) + ModuleProcesser.enable_module_dump = True + self.module_dumper.api_register.register_all_api.reset_mock() + self.module_dumper.stop_module_dump() + self.assertFalse(ModuleProcesser.enable_module_dump) + self.module_dumper.api_register.register_all_api.assert_called_once() + + self.module_dumper.api_register.register_all_api.reset_mock() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_module_processer.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_module_processer.py index f8a561b61b6a758a525675bdc59957e5c923b261..7a1524d0c3ebfb55d0da4652775d890bc27eb45d 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_module_processer.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_module_processer.py @@ -1,104 +1,261 @@ +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest -from unittest.mock import MagicMock +from unittest.mock import patch, MagicMock import torch +import msprobe.pytorch.dump.module_dump.module_processer as mp from msprobe.core.data_dump.scope import ModuleRangeScope -from msprobe.pytorch.common.utils import Const -from msprobe.pytorch.dump.module_dump.module_processer import ModuleProcesser +from msprobe.pytorch.dump.module_dump.module_processer import ( + ModuleProcesser, + replace_checkpoint, + checkpoint_without_early_stop, + wrap_megatron_deallocate +) +ori_checkpoint = torch.utils.checkpoint.checkpoint -class TestModuleProcesser(unittest.TestCase): +class TestWrapper(unittest.TestCase): def setUp(self): - self.mock_tensor = MagicMock(spec=torch.Tensor) + torch.utils.checkpoint.checkpoint = ori_checkpoint + + def test_replace_checkpoint_for_torch_version_above_2(self): + mp.torch_version_above_or_equal_2 = True + with patch('msprobe.pytorch.dump.module_dump.module_processer.checkpoint_without_early_stop') as mock_obj: + replace_checkpoint() + self.assertEqual(torch.utils.checkpoint.checkpoint, mock_obj) + + def test_replace_checkpoint_for_torch_version_below_2(self): + mp.torch_version_above_or_equal_2 = False + replace_checkpoint() + self.assertEqual(torch.utils.checkpoint.checkpoint, ori_checkpoint) + + def test_checkpoint_without_early_stop(self): + mock_checkpoint = MagicMock(return_value="test_result") + + with patch('msprobe.pytorch.dump.module_dump.module_processer.set_checkpoint_early_stop', MagicMock()), \ + patch('msprobe.pytorch.dump.module_dump.module_processer.origin_checkpoint', mock_checkpoint): + result = checkpoint_without_early_stop("input") + mock_checkpoint.assert_called_once_with("input") + self.assertEqual(result, "test_result") + + def test_wrap_megatron_deallocate(self): + mock_func = MagicMock(return_value="output_test") + wrapped = wrap_megatron_deallocate(mock_func) + + mock_tensor = MagicMock(spec=torch.Tensor) + mock_tensor._base = True + mock_tensor.device = "cpu" + mock_tensor.dtype = torch.float32 + mock_tensor.clone.return_value = "cloned" + + result = wrapped(mock_tensor, deallocate_pipeline_outputs=True) + mock_tensor.clone.assert_called_once() + self.assertEqual(mock_tensor.data.shape, (1,)) + self.assertEqual(result, "output_test") + mock_func.assert_called_once_with("cloned", True) + + result = wrapped("normal_input", False) + self.assertEqual(result, "output_test") + mock_func.assert_called_with("normal_input", False) + + +class TestModuleProcesser(unittest.TestCase): + def setUp(self): + ModuleProcesser.module_count = {} + ModuleProcesser.module_stack = [] + ModuleProcesser.module_node = {} + ModuleProcesser.api_parent_node = "" + + self.scope = ModuleRangeScope([], []) self.mock_scope = MagicMock() - self.processor = ModuleProcesser(self.mock_scope) - - def test_scope_is_module_range_scope(self): - scope = ModuleRangeScope([], []) - processor = ModuleProcesser(scope) - self.assertEqual(processor.scope, scope) - - def test_scope_is_not_module_range_scope(self): - scope = "not a ModuleRangeScope" - processor = ModuleProcesser(scope) - self.assertIsNone(processor.scope) - - def test_clone_return_value_and_test_clone_if_tensor(self): - def func(x): - return x - - input = torch.tensor([1]) - input_tuple = (torch.tensor([1]), torch.tensor([2])) - input_list = [torch.tensor([1]), torch.tensor([2])] - input_dict = {"A": torch.tensor([1]), "B": torch.tensor([2])} - - result = ModuleProcesser.clone_return_value(func)(input) - result[0] = 2 - self.assertNotEqual(result, input) - result_tuple = ModuleProcesser.clone_return_value(func)(input_tuple) - result_tuple[0][0] = 2 - self.assertNotEqual(result_tuple, input_tuple) - result_list = ModuleProcesser.clone_return_value(func)(input_list) - result_list[0][0] = 2 - self.assertNotEqual(result_list, input_list) - result_dict = ModuleProcesser.clone_return_value(func)(input_dict) - result_dict["A"][0] = 2 - self.assertNotEqual(result_dict, input_dict) - - def test_module_count_func(self): - test = ModuleProcesser(None) - self.assertEqual(test.module_count, {}) - module_name = "nope" - test.module_count_func(module_name) - self.assertEqual(test.module_count["nope"], 0) - - def test_node_hook_forward_start(self): - name_prefix = "forward_layer" - hook = self.processor.node_hook(name_prefix, start_or_stop=Const.START) - module = MagicMock() - input = (self.mock_tensor,) - module.mindstudio_reserved_name = None - hook(module, input) - expected_name = f"forward_layer{Const.SEP}0" - self.assertEqual(module.mindstudio_reserved_name, [expected_name]) - self.assertIn(expected_name, ModuleProcesser.module_stack) - self.assertEqual(ModuleProcesser.api_parent_node, expected_name) - - def test_node_hook_forward_stop(self): - name_prefix = "forward_layer" - hook = self.processor.node_hook(name_prefix, start_or_stop=Const.STOP) - ModuleProcesser.module_stack.append(f"forward_layer{Const.SEP}0") - - module = MagicMock() - input = (self.mock_tensor,) - reserved_name = f"forward_layer{Const.SEP}0" - module.mindstudio_reserved_name = [reserved_name] - hook(module, input) - self.assertNotIn([f"forward_layer{Const.SEP}0"], ModuleProcesser.module_stack) - self.assertEqual(ModuleProcesser.api_parent_node, reserved_name) - - def test_node_hook_backward(self): - name_prefix = "backward_layer" - hook = self.processor.node_hook(name_prefix, start_or_stop=Const.START) - - module = MagicMock() - input = (self.mock_tensor,) - module.mindstudio_reserved_name = None - ModuleProcesser.module_node[f"forward_layer{Const.SEP}0"] = None - hook(module, input) - expected_name = f"backward_layer{Const.SEP}0" - self.assertEqual(module.mindstudio_reserved_name, [expected_name]) - self.assertIn(expected_name, ModuleProcesser.module_node) + + @patch('msprobe.pytorch.dump.module_dump.module_processer.wrap_setup_input_output_hook') + @patch('msprobe.pytorch.dump.module_dump.module_processer.replace_checkpoint') + def test_init_with_valid_scope(self, mock_replace, mock_wrap): + processor = ModuleProcesser(self.scope) + self.assertEqual(processor.scope, self.scope) + mock_wrap.assert_called_once() + mock_replace.assert_called_once() + + @patch('msprobe.pytorch.dump.module_dump.module_processer.logger.info_on_rank_0') + def test_init_without_megatron(self, mock_log): + ModuleProcesser(self.scope) + mock_log.assert_called_with("No megatron find.") + + def test_set_and_get_calls_number(self): + count = ModuleProcesser.set_and_get_calls_number("test_module") + self.assertEqual(count, 0) + + count = ModuleProcesser.set_and_get_calls_number("test_module") + self.assertEqual(count, 1) def test_has_register_backward_hook(self): - module = MagicMock() - module._backward_hooks = {0: lambda: None} - module._is_full_backward_hook = False - result = self.processor.has_register_backward_hook(module) - self.assertTrue(result) - - module._is_full_backward_hook = True - result = self.processor.has_register_backward_hook(module) - self.assertFalse(result) + module1 = torch.nn.Linear(10, 10) + self.assertFalse(ModuleProcesser.has_register_backward_hook(module1)) + + module2 = MagicMock() + module2._backward_hooks = [1, 2, 3] + module2._is_full_backward_hook = False + self.assertTrue(ModuleProcesser.has_register_backward_hook(module2)) + + module2._is_full_backward_hook = True + self.assertFalse(ModuleProcesser.has_register_backward_hook(module2)) + + def test_get_modules_and_names_with_model_list(self): + mock_model1 = MagicMock() + mock_model2 = MagicMock() + mock_model1.named_modules.return_value = [("layer1", "obj1"), ("layer2", "obj2")] + mock_model2.named_modules.return_value = [("layer3", "obj3")] + + result = ModuleProcesser.get_modules_and_names( + [mock_model1, mock_model2], + recursive=True, + module_names=["model1", "model2"] + ) + self.assertEqual(result, { + "0": [("layer1", "obj1"), ("layer2", "obj2")], + "1": [("layer3", "obj3")] + }) + + def test_get_modules_and_names_with_model_tuple(self): + mock_model1 = MagicMock() + mock_model2 = MagicMock() + mock_model1.named_modules.return_value = [("layer1", "obj1")] + mock_model2.named_modules.return_value = [("layer2", "obj2")] + + result = ModuleProcesser.get_modules_and_names( + (mock_model1, mock_model2), + recursive=True, + module_names=["model1", "model2"] + ) + self.assertEqual(result, { + "0": [("layer1", "obj1")], + "1": [("layer2", "obj2")] + }) + + def test_get_modules_and_names_with_single_recursive(self): + mock_model = MagicMock() + mock_model.named_modules.return_value = [("layer1", "obj1")] + + result = ModuleProcesser.get_modules_and_names( + mock_model, + recursive=True, + module_names=["single_model"] + ) + self.assertEqual(result, { + "-1": [("layer1", "obj1")] + }) + + def test_get_modules_and_names_with_single_non_recursive(self): + mock_model = MagicMock() + result = ModuleProcesser.get_modules_and_names( + mock_model, + recursive=False, + module_names=["single_model"] + ) + self.assertEqual(result, { + "-1": [("single_model", mock_model)] + }) + + def test_get_modules_and_names_invalid_case(self): + result = ModuleProcesser.get_modules_and_names( + [MagicMock(), MagicMock()], + recursive=False, + module_names=["only_one_name"] + ) + self.assertEqual(result, {}) + + result = ModuleProcesser.get_modules_and_names( + MagicMock(), + recursive=False, + module_names=["name1", "name2"] + ) + self.assertEqual(result, {}) + + def test_reset_module_stats(self): + ModuleProcesser.module_count = {"test": 1} + ModuleProcesser.module_stack = ["layer1"] + ModuleProcesser.api_parent_node = "parent" + ModuleProcesser.module_node = {"key": "value"} + ModuleProcesser.module_bw_hook_kernels = {"hook": "data"} + ModuleProcesser.enable_module_dump = True + + ModuleProcesser.reset_module_stats() + + self.assertEqual(ModuleProcesser.module_count, {}) + self.assertEqual(ModuleProcesser.module_stack, []) + self.assertEqual(ModuleProcesser.api_parent_node, "") + self.assertEqual(ModuleProcesser.module_node, {}) + self.assertEqual(ModuleProcesser.module_bw_hook_kernels, {}) + self.assertFalse(ModuleProcesser.enable_module_dump) + + def test_set_construct_info_in_pre_hook_with_stack(self): + processor = ModuleProcesser(self.mock_scope) + ModuleProcesser.module_stack = ["parent_module"] + processor.scope = self.mock_scope + + processor.set_construct_info_in_pre_hook("current_module") + + self.assertEqual(ModuleProcesser.module_node["current_module"], "parent_module") + self.assertEqual(ModuleProcesser.module_stack, ["parent_module", "current_module"]) + self.assertEqual(ModuleProcesser.api_parent_node, "current_module") + self.mock_scope.begin_module.assert_called_once_with("current_module") + + def test_set_construct_info_in_pre_hook_empty_stack(self): + processor = ModuleProcesser(self.mock_scope) + processor.scope = self.mock_scope + processor.set_construct_info_in_pre_hook("root_module") + + self.assertIsNone(ModuleProcesser.module_node["root_module"]) + self.assertEqual(ModuleProcesser.module_stack, ["root_module"]) + self.assertEqual(ModuleProcesser.api_parent_node, "root_module") + + def test_set_construct_info_in_hook_with_forward(self): + mp.torch_version_above_or_equal_2 = True + processor = ModuleProcesser(self.mock_scope) + ModuleProcesser.module_stack = ["parent", "current"] + processor.scope = self.mock_scope + + processor.set_construct_info_in_hook("current") + + self.assertEqual(ModuleProcesser.module_stack, ["parent"]) + self.assertEqual(ModuleProcesser.api_parent_node, "parent") + self.mock_scope.end_module.assert_called_once_with("current") + + def test_set_construct_info_in_hook_with_backward(self): + mp.torch_version_above_or_equal_2 = False + processor = ModuleProcesser(self.mock_scope) + processor.scope = self.mock_scope + + processor.set_construct_info_in_hook("backward_module", is_forward=False) + + self.assertEqual(ModuleProcesser.api_parent_node, "backward_module") + self.mock_scope.begin_module.assert_called_once_with("backward_module") + + def test_set_construct_info_in_hook_empty_stack(self): + mp.torch_version_above_or_equal_2 = True + processor = ModuleProcesser(self.mock_scope) + + processor.set_construct_info_in_hook("module") + + self.assertIsNone(ModuleProcesser.api_parent_node) + + +if __name__ == "__main__": + unittest.main() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_pt_hook_wrapper.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_pt_hook_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..88039390f1900bde2e81390af778b8f83c7eb8ff --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/dump/test_pt_hook_wrapper.py @@ -0,0 +1,92 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest.mock import MagicMock, patch + +import torch + +from msprobe.pytorch.dump.module_dump.hook_wrapper import wrap_setup_backward_hook + + +class TestWrapSetupBackwardHook(unittest.TestCase): + def setUp(self): + self.mock_func = MagicMock() + self.mock_func.return_value = ["clone_tensor1", "clone_tensor2"] + + self.decorated_func = wrap_setup_backward_hook(self.mock_func) + + self.tensor = torch.randn(3, requires_grad=True) + torch.set_grad_enabled(True) + + def test_insufficient_args(self): + result = self.decorated_func("test_case1") + self.mock_func.assert_called_once_with("test_case1") + self.assertListEqual(result, ["clone_tensor1", "clone_tensor2"]) + + def test_normal_processing_flow(self): + test_tensor = torch.randn(2, requires_grad=False) + test_data = { + "tensors": [self.tensor, torch.randn(2, requires_grad=True)], + "nested": { + "tuple": (self.tensor, test_tensor) + } + } + + mock_self = MagicMock() + mock_self.module.inplace = False + test_tensor1 = torch.randn(4, requires_grad=True) + test_tensor2 = torch.randn(4, requires_grad=True) + test_tensor3 = torch.randn(4, requires_grad=True) + self.mock_func.return_value = [test_tensor1, test_tensor2, test_tensor3] + result = self.decorated_func(mock_self, test_data) + + self.assertIsInstance(result, dict) + self.assertFalse(torch.equal(result["tensors"][0], self.tensor)) + self.assertTrue(torch.equal(result["tensors"][1], test_tensor2)) + self.assertIsInstance(result["nested"]["tuple"][0], torch.Tensor) + self.assertTrue(torch.equal(result["nested"]["tuple"][1], test_tensor)) + + def test_complex_data_structures(self): + test_case = [ + self.tensor, + {"dict": torch.randn(4, requires_grad=True)}, + (torch.randn(5, requires_grad=True),), + [torch.randn(6, requires_grad=True)] + ] + + mock_self = MagicMock() + mock_self.module.inplace = False + test_tensor1 = torch.randn(4, requires_grad=True) + test_tensor2 = torch.randn(5, requires_grad=True) + test_tensor3 = torch.randn(6, requires_grad=True) + self.mock_func.return_value = [self.tensor, test_tensor1, test_tensor2, test_tensor3] + result = self.decorated_func(mock_self, test_case) + + self.assertIsInstance(result, list) + self.assertTrue(torch.equal(result[1]["dict"], test_tensor1)) + self.assertTrue(torch.equal(result[2][0], test_tensor2)) + self.assertTrue(torch.equal(result[3][0], test_tensor3)) + + @patch('msprobe.pytorch.common.utils.is_float8_tensor', return_value=True) + def test_float8_tensor_handling(self, _): + test_data = [torch.randn(3, requires_grad=True)] + + mock_self = MagicMock() + self.mock_func.return_value = [] + result = self.decorated_func(mock_self, test_data) + + self.assertIsInstance(result, list) + self.assertListEqual(result, test_data) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/perturbed_layers/test_perturbed_layser.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/perturbed_layers/test_perturbed_layser.py index be2215dcd9cb22577a84954b9283ed68825de86e..d4e568303a8b22058cba4ad879b160b3169a6cae 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/perturbed_layers/test_perturbed_layser.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/perturbed_layers/test_perturbed_layser.py @@ -166,7 +166,7 @@ class TestPerturbedLayer(TestCase): layer.pre_check(y) mock_logger.assert_called_with( "[msprobe] Free Benchmark: For test_api_name, " - "Maximun value is less than the minimun threshold. Cancel add noise." + "maximum value is less than the minimum threshold. Cancel adding noise." ) # 对于输入张量，add_noise扰动因子对大于极小值的部分增加一个小值 @@ -212,7 +212,7 @@ class TestPerturbedLayer(TestCase): layer.pre_check(y) mock_logger.assert_called_with( "[msprobe] Free Benchmark: For test_api_name, " - "Maximun value is less than the minimun threshold. Cancel add noise." + "maximum value is less than the minimum threshold. Cancel adding noise." ) # 对于低精度输入、run cpu会升精度在cpu上计算，并会打印日志 diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_hook_module.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_hook_module.py deleted file mode 100644 index 1524a82ae1fc81eee245fa73bde4b4938cb89638..0000000000000000000000000000000000000000 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_hook_module.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -from unittest.mock import MagicMock, patch -import threading -from msprobe.pytorch.hook_module.hook_module import HOOKModule - -class TestHOOKModuleInit(unittest.TestCase): - - def setUp(self): - self.mock_build_hook = MagicMock(return_value=(MagicMock(), MagicMock(), MagicMock(), None)) - - def test_thread_handling(self): - module = HOOKModule(self.mock_build_hook) - current_thread_id = module.current_thread - self.assertEqual(current_thread_id, threading.current_thread().ident) - - -class TestHOOKModuleCall(unittest.TestCase): - def setUp(self): - self.mock_build_hook = MagicMock(return_value=(MagicMock(), MagicMock(), MagicMock(), None)) - self.module = HOOKModule(self.mock_build_hook) - - @patch.object(HOOKModule, '_call_func') - def test_call_function(self, mock_call_func): - mock_call_func.return_value = "test_result" - result = self.module("input_data") - mock_call_func.assert_called_once_with("input_data", **{}) - self.assertEqual(result, "test_result") - - @patch.object(HOOKModule, '_call_func') - def test_call_func_with_hooks(self, mock_call_func): - mock_call_func.return_value = "test_result_with_hooks" - result = self.module("input_data") - self.assertEqual(result, "test_result_with_hooks") - HOOKModule.inner_stop_hook[self.module.current_thread] = False diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_api_register.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_api_register.py new file mode 100644 index 0000000000000000000000000000000000000000..bb2091c342b7d2036a96c1ccabeb1f81a49b65b2 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_api_register.py @@ -0,0 +1,224 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest.mock import MagicMock, patch + +import msprobe.pytorch.hook_module.api_register as api_register +from msprobe.pytorch.hook_module.api_register import ( + tensor_module_forward, + dist_module_forward, + npu_module_forward, + get_api_register, + ApiTemplate +) + + +class TestAPIRegister(unittest.TestCase): + def setUp(self): + api_register.api_register = None + + def test_tensor_module_forward(self): + mock_module = MagicMock() + mock_module.api_name = "test_name" + mock_module.api_func.return_value = "test_result" + + args = (1, 2, 3) + kwargs = {"key": "value"} + result = tensor_module_forward(mock_module, *args, **kwargs) + + mock_module.api_func.assert_called_once_with(*args, **kwargs) + self.assertEqual(result, "test_result") + + @patch('msprobe.pytorch.hook_module.api_register.logger.warning') + def test_basic_dist_module_forward(self, mock_logger): + mock_module = MagicMock() + mock_module.api_func.return_value = "test_handle" + mock_module.api_name = "test_api" + + result = dist_module_forward(mock_module, 1, 2, key="value") + mock_module.api_func.assert_called_once_with(1, 2, key="value") + self.assertEqual(result, "test_handle") + mock_logger.assert_not_called() + + def test_dist_module_forward_with_batch_op(self): + mock_reqs = [MagicMock(), MagicMock()] + mock_module = MagicMock() + mock_module.api_func.return_value = mock_reqs + mock_module.api_name = "batch_isend_irecv" + + result = dist_module_forward(mock_module) + + for req in mock_reqs: + req.wait.assert_called_once() + self.assertEqual(result, mock_reqs) + + @patch('msprobe.pytorch.hook_module.api_register.ApiRegistry') + def test_get_api_register_with_new_obj(self, mock_api_registry): + get_api_register(return_new=True) + mock_api_registry.assert_called_once() + self.assertIsNone(api_register.api_register) + + @patch('msprobe.pytorch.hook_module.api_register.ApiRegistry') + def test_get_api_register_with_not_new_obj(self, mock_api_registry): + get_api_register() + mock_api_registry.assert_called_once() + self.assertIsNotNone(api_register.api_register) + + +class TestNpuModuleForward(unittest.TestCase): + def setUp(self): + self.npu_custom_functions = { + "custom_func": MagicMock(return_value="custom_result"), + "npu_fusion_attention": MagicMock(return_value="nfa_result"), + "gpu_fusion_attention": MagicMock(return_value="gfa_result") + } + + self.module = MagicMock() + self.module.api_func.return_value = "test_result" + + def test_with_hook_enabled(self): + self.module.need_hook = True + result = npu_module_forward(self.module, 1, 2, key="value") + self.module.api_func.assert_called_once_with(1, 2, key="value") + self.assertEqual(result, "test_result") + + def test_with_unknown_api(self): + self.module.need_hook = False + self.module.api_name = "unknown_func" + with patch('msprobe.pytorch.hook_module.api_register.npu_custom_functions', new=self.npu_custom_functions): + with self.assertRaises(Exception) as context: + npu_module_forward(self.module, 1, 2, key="value") + self.assertIn("There is not bench function unknown_func", str(context.exception)) + + def test_cuda_device_with_mapping(self): + self.module.need_hook = False + self.module.api_name = "npu_fusion_attention" + self.module.device = 'cuda' + + with patch('msprobe.pytorch.hook_module.api_register.npu_custom_functions', new=self.npu_custom_functions): + result = npu_module_forward(self.module, 1, 2, key="value") + self.npu_custom_functions["gpu_fusion_attention"].assert_called_once_with(1, 2, key="value") + self.assertEqual(result, "gfa_result") + + def test_cpu_device(self): + self.module.need_hook = False + self.module.api_name = "custom_func" + self.module.device = "cpu" + + with patch('msprobe.pytorch.hook_module.api_register.npu_custom_functions', new=self.npu_custom_functions): + result = npu_module_forward(self.module, 1, 2, key="value") + self.npu_custom_functions["custom_func"].assert_called_once_with(1, 2, key="value") + self.assertEqual(result, "custom_result") + + def test_unsupported_device(self): + self.module.need_hook = False + self.module.api_name = "custom_func" + self.module.device = "unsupported_device" + + with patch('msprobe.pytorch.hook_module.api_register.npu_custom_functions', new=self.npu_custom_functions): + result = npu_module_forward(self.module, 1, 2, key="value") + self.module.api_func.assert_called_once_with(1, 2, key="value") + self.assertEqual(result, "test_result") + + +class TestApiTemplate(unittest.TestCase): + def setUp(self): + self.api_name = "Tensor.test_api" + self.api_func = MagicMock(return_value="test_result") + self.prefix = "test_prefix" + self.hook_build_func = MagicMock() + self.mock_hook_module = MagicMock() + + def test_init(self): + with patch('msprobe.pytorch.hook_module.api_register.HOOKModule') as mock_hook_module: + template = ApiTemplate( + self.api_name, + self.api_func, + self.prefix, + self.hook_build_func, + need_hook=False + ) + + self.assertEqual(template.api_name, self.api_name) + self.assertEqual(template.api_func, self.api_func) + self.assertEqual(template.prefix, self.prefix) + self.assertEqual(template.prefix_api_name, "test_prefix.test_api.") + self.assertEqual(template.device, "cpu") + self.assertFalse(template.need_hook) + + self.assertFalse(hasattr(template, 'op_is_distributed')) + + def test_init_with_distributed_prefix(self): + with patch('msprobe.pytorch.hook_module.api_register.HOOKModule'): + self.prefix = "Distributed" + template = ApiTemplate( + self.api_name, + self.api_func, + self.prefix, + self.hook_build_func, + need_hook=False, + device="npu" + ) + + self.assertEqual(template.device, "npu") + self.assertEqual(template.prefix_api_name, "Distributed.test_api.") + self.assertTrue(template.op_is_distributed) + + def test_init_without_hook(self): + with patch('msprobe.pytorch.hook_module.api_register.HOOKModule') as mock_hook_module: + template = ApiTemplate( + self.api_name, + self.api_func, + self.prefix, + self.hook_build_func, + need_hook=False, + device="npu" + ) + + self.assertFalse(template.need_hook) + self.mock_hook_module.assert_not_called() + + def test_forward_with_prefix_match(self): + with patch('msprobe.pytorch.hook_module.api_register.HOOKModule'): + self.prefix = "Tensor" + template = ApiTemplate( + self.api_name, + self.api_func, + self.prefix, + self.hook_build_func, + need_hook=False, + device="npu" + ) + + result = template.forward("arg1", key="value") + + self.assertEqual(result, "test_result") + + def test_forward_without_prefix_match(self): + with patch('msprobe.pytorch.hook_module.api_register.HOOKModule'): + template = ApiTemplate( + self.api_name, + self.api_func, + self.prefix, + self.hook_build_func, + need_hook=False, + device="npu" + ) + + result = template.forward("arg1", key="value") + + self.api_func.assert_called_once_with("arg1", key="value") + self.assertEqual(result, "test_result") diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_hook_module.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_hook_module.py new file mode 100644 index 0000000000000000000000000000000000000000..b66e6b94900d461feb98fb029e6560ec44758ef1 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_hook_module.py @@ -0,0 +1,91 @@ +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading +import unittest +from collections import defaultdict +from unittest.mock import MagicMock, patch + +from msprobe.pytorch.hook_module.hook_module import HOOKModule + + +class TestHOOKModule(unittest.TestCase): + def setUp(self): + self.mock_build_hook = MagicMock(return_value=(MagicMock(), MagicMock(), MagicMock())) + HOOKModule.module_count = defaultdict(int) + HOOKModule.inner_stop_hook = {} + + def test_init_with_stop_hook(self): + expected_thread = threading.current_thread().ident + HOOKModule.inner_stop_hook[expected_thread] = True + + module1 = HOOKModule(self.mock_build_hook) + current_thread = module1.current_thread + + self.assertEqual(current_thread, expected_thread) + self.assertTrue(module1.inner_stop_hook[current_thread]) + self.assertTrue(module1.stop_hook) + self.assertFalse(hasattr(module1, "forward_data_collected")) + + def test_init_with_start_hook(self): + module1 = HOOKModule(self.mock_build_hook) + current_thread = module1.current_thread + expected_thread = threading.current_thread().ident + + self.assertEqual(current_thread, expected_thread) + self.assertFalse(module1.inner_stop_hook[current_thread]) + self.assertFalse(module1.stop_hook) + self.assertTrue(hasattr(module1, "forward_data_collected")) + + @patch.object(HOOKModule, '_call_func') + def test_call_with_stop_hooks(self, mock_call_func): + mock_call_func.return_value = "test_result" + expected_thread = threading.current_thread().ident + HOOKModule.inner_stop_hook[expected_thread] = True + + module1 = HOOKModule(self.mock_build_hook) + self.assertTrue(module1.stop_hook) + + result = module1("arg1", "arg2", key="value") + mock_call_func.assert_called_once_with("arg1", "arg2", key="value") + self.assertEqual(result, "test_result") + self.assertTrue(HOOKModule.inner_stop_hook[expected_thread]) + + @patch.object(HOOKModule, '_call_func') + def test_call_with_start_hooks(self, mock_call_func): + mock_call_func.return_value = "test_result" + expected_thread = threading.current_thread().ident + + module1 = HOOKModule(self.mock_build_hook) + self.assertFalse(module1.stop_hook) + + result = module1("arg1", "arg2", key="value") + mock_call_func.assert_called_once_with("arg1", "arg2", key="value") + self.assertEqual(result, "test_result") + self.assertFalse(HOOKModule.inner_stop_hook[expected_thread]) + + def test_reset_module_stats(self): + HOOKModule.module_count = {"Tensor.add.0.forward": 0} + HOOKModule.reset_module_stats() + self.assertDictEqual(HOOKModule.module_count, defaultdict(int)) + + def test_add_module_count(self): + HOOKModule.add_module_count("Tensor.add.0.forward") + self.assertEqual(HOOKModule.module_count["Tensor.add.0.forward"], 1) + + def test_get_module_count(self): + HOOKModule.module_count = {"Tensor.add.0.forward": 0} + result = HOOKModule.get_module_count("Tensor.add.0.forward") + self.assertEqual(result, 0) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_hook_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_hook_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..de70f8a11fb789189b56495c1ae9d1bad977fb71 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_hook_utils.py @@ -0,0 +1,80 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest +from unittest.mock import MagicMock, patch + +from msprobe.pytorch.hook_module.utils import get_ops, dynamic_import_op + + +class MockPackage: + __name__ = "mock_package" + __file__ = "/fake_path/__init__.py" + + +class TestUtils(unittest.TestCase): + def setUp(self): + self.yaml_content = { + 'functional': ['func1', 'func2'], + 'tensor': ['tensor_op1'], + 'torch': ['torch_op1', 'torch_op2'], + 'torch_npu': ['npu_op1'] + } + + self.mock_listdir = patch('os.listdir').start() + self.mock_check_link = patch('msprobe.pytorch.hook_module.utils.check_link').start() + + def tearDown(self): + patch.stopall() + + def test_get_ops(self): + with patch('msprobe.pytorch.hook_module.utils.load_yaml') as mock_load: + mock_load.return_value = self.yaml_content + result = get_ops() + self.assertEqual( + result, + { + 'func1', + 'func2', + 'tensor_op1', + 'torch_op1', + 'torch_op2', + 'npu_op1' + } + ) + + @patch('msprobe.pytorch.hook_module.utils.inspect') + def test_dynamic_import_op_success(self, mock_inspect): + mock_func = lambda x: x + mock_inspect.getmembers = MagicMock() + mock_inspect.getmembers.return_value = [['test_func', mock_func]] + + self.mock_listdir.return_value = ['valid.py', 'invalid.py'] + mock_module = MagicMock() + + with patch('importlib.import_module', return_value=mock_module) as mock_import: + ops = dynamic_import_op(MockPackage(), white_list=['valid.py']) + self.assertEqual(ops, {'valid.test_func': mock_func}) + mock_import.assert_called_once_with('mock_package.valid') + + def test_dynamic_import_op_failure(self): + self.mock_listdir.return_value = ['fail.py'] + with patch('importlib.import_module') as mock_import: + mock_import.side_effect = ImportError("Fake error") + with patch('msprobe.pytorch.hook_module.utils.logger.warning') as mock_logger: + ops = dynamic_import_op(MockPackage(), white_list=['fail.py']) + self.assertEqual(ops, {}) + mock_logger.assert_called_once_with("import mock_package.fail failed!") \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_jit_script_wrapper.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_jit_script_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..61909523fc523dead62887ba94f399424b72a098 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_pt_jit_script_wrapper.py @@ -0,0 +1,51 @@ +# Copyright (c) 2025-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +from unittest.mock import MagicMock, patch + +import torch +from msprobe.pytorch.hook_module.jit_script_wrapper import wrap_jit_script_func + + +class TestWrapJitScriptFunc(unittest.TestCase): + def setUp(self): + self.original_script = torch.jit.script + + self.mock_api_register = MagicMock() + self.mock_api_register.all_api_registered = True + self.mock_api_register.register_all_api = MagicMock() + self.mock_api_register.restore_all_api = MagicMock() + + def tearDown(self): + torch.jit.script = self.original_script + + @patch('torch.jit.script', new_callable=MagicMock) + @patch('msprobe.pytorch.hook_module.jit_script_wrapper.get_api_register', return_value=MagicMock()) + def test_patched_script(self, mock_get_api, mock_original_script): + mock_original_script.return_value = "mocked_result" + mock_get_api.return_value = self.mock_api_register + + wrap_jit_script_func() + + self.assertNotEqual(torch.jit.script, self.original_script) + + result = torch.jit.script("test_input") + + mock_original_script.assert_called_once_with("test_input") + self.assertEqual(result, "mocked_result") + + self.mock_api_register.restore_all_api.assert_called_once() + self.mock_api_register.register_all_api.assert_called_once() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py index af669cb5c73de85e51f36f62f9e7dc61bb599ca1..e565c1cc08d496bd96cc1e873f50e4c02e5c69a8 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py @@ -1,15 +1,34 @@ +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import unittest from unittest.mock import MagicMock, patch import torch from msprobe.pytorch.function_factory import npu_custom_grad_functions -from msprobe.pytorch.hook_module.wrap_aten import AtenOPTemplate, white_aten_ops, \ +from msprobe.pytorch.hook_module.wrap_aten import ( + AtenOPTemplate, + white_aten_ops, AtenOPPacketTemplate +) def mock_build_hook(prefix): - return (MagicMock(), MagicMock(), MagicMock(), MagicMock()) + return (MagicMock(), MagicMock(), MagicMock()) + class TestAtenOPTemplate(unittest.TestCase): @@ -79,8 +98,8 @@ class TestAtenOPPacketTemplate(unittest.TestCase): del self.mock_op_packet.nonexistent_attr with self.assertRaises(AttributeError) as context: _ = self.template.nonexistent_attr - self.assertIn("or OpOverloadPacket does not have attribute 'nonexistent_attr'.", \ - str(context.exception)) + self.assertIn("or OpOverloadPacket does not have attribute 'nonexistent_attr'.", + str(context.exception)) @patch('msprobe.pytorch.hook_module.wrap_aten.AtenOPTemplate', autospec=True) def test_getattr_op_overload(self, MockAtenOPTemplate): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/config/stack_config.json b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/config/stack_config.json new file mode 100644 index 0000000000000000000000000000000000000000..461b447ce0cd33fdcbab3476f7c1e3bcdee9dfad --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/config/stack_config.json @@ -0,0 +1,5 @@ +{ + "targets": {}, + "format": "csv", + "stack_info": true +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_anomaly_analyse.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_anomaly_analyse.py index 904be210a3771f1757e4410b5e0fa0f2ad6152f2..ad4a97acaa9940e807e4023b9745bd210a827501 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_anomaly_analyse.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_anomaly_analyse.py @@ -42,7 +42,6 @@ class TestAnomalyDataWriter(unittest.TestCase): writer.init_detected_json() # 检查是否创建了目录 - mock_create_directory.assert_any_call('/tmp/dump') mock_create_directory.assert_any_call('/tmp/dump/rank0') # 检查是否初始化了 JSON 文件 diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_anomaly_detect.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_anomaly_detect.py index fa0960e2cc1842a138b47fad3f86c1ed0d089db8..6e416de8c689e6df7642dd52c60021a7e1b58baf 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_anomaly_detect.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_anomaly_detect.py @@ -2,7 +2,7 @@ import unittest from unittest import TestCase from unittest.mock import patch -from msprobe.pytorch.monitor.anomaly_detect import AnomalyTurbulence, AnomalyScanner, \ +from msprobe.pytorch.monitor.anomaly_detect import AnomalyTurbulence, AnomalyNan, AnomalyScanner, \ AnomalyDataFactory, GradAnomalyData, BaseWriterWithAD, ScanRule, WriterInput @@ -24,15 +24,43 @@ class TestAnomalyTurbulence(TestCase): def test_apply_with_positive_baseline(self): history = [10, 12, 14] cur = 16 - result = self.rule.apply(history, cur) + result = self.rule.apply(cur, history=history) self.assertTrue(result) def test_apply_with_non_positive_baseline(self): history = [0, 0, 0] cur = -1 - result = self.rule.apply(history, cur) + result = self.rule.apply(cur, history=history) self.assertTrue(result) + def test_apply_with_valid_value(self): + history = [0, 0, 0] + cur = 0 + result = self.rule.apply(cur, history=history) + self.assertFalse(result) + + +class TestAnomalyNan(TestCase): + + def setUp(self) -> None: + self.threshold = 1e10 + self.rule = AnomalyNan(self.threshold) + + def test_apply_with_nan(self): + cur = float("nan") + result = self.rule.apply(cur) + self.assertTrue(result) + + def test_apply_with_big_value(self): + cur = float("1e30") + result = self.rule.apply(cur) + self.assertTrue(result) + + def test_apply_with_valid_value(self): + cur = 0.5 + result = self.rule.apply(cur) + self.assertFalse(result) + class TestAnomalyScanner(TestCase): @@ -266,8 +294,9 @@ class TestBaseWriterWithAD(TestCase): def test_add_scalar(self, mock_logger): AnomalyTurbulence_obj = AnomalyTurbulence(0.2) self.BaseWriter.ad_rules = [AnomalyTurbulence_obj] - self.BaseWriter.tag2scalars = {'tag': {'avg': 1.0, 'count': 1}} - self.BaseWriter.add_scalar('tag', 2.0) + tag = ('0:1.post_attention_norm.weight/rank0/pre_grad', 'mean') + self.BaseWriter.tag2scalars = {tag: {'avg': 1.0, 'count': 1}} + self.BaseWriter.add_scalar(tag, 2.0) mock_logger.info.assert_called_once() @@ -283,7 +312,7 @@ class TestBaseWriterWithAD(TestCase): self.assertEqual(self.BaseWriter.tag2scalars['tag1']['avg'], 1.0) self.assertEqual(self.BaseWriter.tag2scalars['tag1']['count'], 1) self.BaseWriter._update_tag2scalars('tag1', 2.0) - self.assertEqual(self.BaseWriter.tag2scalars['tag1']['avg'], 1.5) + self.assertEqual(self.BaseWriter.tag2scalars['tag1']['avg'], 1.01) self.assertEqual(self.BaseWriter.tag2scalars['tag1']['count'], 2) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_csv2tb.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_csv2tb.py index 4178e2ef8fbfb2c2bafa90b32fa92d622b95e3cd..09e860e7ac5048bd059f888eabfd8ad1d7f45d37 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_csv2tb.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_csv2tb.py @@ -17,7 +17,6 @@ import os import shutil import random import unittest -import pytest import torch import numpy as np import torch.nn as nn @@ -30,13 +29,9 @@ from msprobe.pytorch.hook_module.api_register import get_api_register get_api_register().restore_all_api() - base_dir = os.path.dirname(os.path.realpath(__file__)) config_json_path = os.path.join(base_dir, "config", "all_config.json") monitor_output = os.path.join(base_dir, "./monitor_output_csv2tb") -os.environ[MonitorConst.MONITOR_OUTPUT_DIR] = monitor_output -timestamp_dirpath = None -csv2tb_dirpath = None def seed_all(seed=1234, mode=False): @@ -46,8 +41,8 @@ def seed_all(seed=1234, mode=False): torch.manual_seed(seed) torch.use_deterministic_algorithms(mode) -seed_all() +seed_all() inputs = [torch.rand(10, 10) for _ in range(10)] labels = [torch.randint(0, 5, (10,)) for _ in range(10)] @@ -65,31 +60,6 @@ class MockModule(nn.Module): return x2 -def data_collect(): - loss_fun = nn.CrossEntropyLoss() - test_module = MockModule() - nn.init.constant_(test_module.linear.weight, 1.0) - nn.init.constant_(test_module.linear.bias, 1.0) - optimizer = torch.optim.Adam(test_module.parameters()) - - monitor = TrainerMon(config_json_path, params_have_main_grad=False) - monitor.set_monitor(test_module, grad_acc_steps=1, optimizer=optimizer) - - for input_data, label in zip(inputs, labels): - output = test_module(input_data) - loss = loss_fun(output, label) - optimizer.zero_grad() - loss.backward() - optimizer.step() - - global timestamp_dirpath, csv2tb_dirpath - timestamp_dirpath = os.path.join(monitor_output, os.listdir(monitor_output)[0]) - csv2tensorboard_by_step(monitor_output) - for dirname in os.listdir(monitor_output): - if "csv2tensorboard" in dirname: - csv2tb_dirpath = os.path.join(monitor_output, dirname, "rank0") - - def extract_scalars_from_tensorboard(log_dir): # 初始化 EventAccumulator event_acc = EventAccumulator(log_dir) @@ -144,97 +114,102 @@ def compare_scalar_dicts(dict1, dict2): return True -@pytest.fixture(scope="session") -def setup_all(): - data_collect() - yield - shutil.rmtree(monitor_output) - -@pytest.mark.usefixtures("setup_all") class TestGradMonitor(unittest.TestCase): + timestamp_dirpath = None + csv2tb_dirpath = None + + @classmethod + def setUpClass(cls): + + os.environ[MonitorConst.MONITOR_OUTPUT_DIR] = monitor_output + if os.path.exists(monitor_output): + shutil.rmtree(monitor_output) + + loss_fun = nn.CrossEntropyLoss() + test_module = MockModule() + nn.init.constant_(test_module.linear.weight, 1.0) + nn.init.constant_(test_module.linear.bias, 1.0) + optimizer = torch.optim.Adam(test_module.parameters()) + + monitor = TrainerMon(config_json_path, params_have_main_grad=False) + monitor.set_monitor(test_module, grad_acc_steps=1, optimizer=optimizer) + + for input_data, label in zip(inputs, labels): + output = test_module(input_data) + loss = loss_fun(output, label) + optimizer.zero_grad() + loss.backward() + optimizer.step() + + cls.timestamp_dirpath = os.path.join(monitor_output, os.listdir(monitor_output)[0]) + csv2tensorboard_by_step(monitor_output) + for dirname in os.listdir(monitor_output): + if "csv2tensorboard" in dirname: + cls.csv2tb_dirpath = os.path.join(monitor_output, dirname, "rank0") + os.environ.pop(MonitorConst.MONITOR_OUTPUT_DIR) def setUp(self): self.maxDiff = None - + def test_actv(self): - data = parse_step_fn(os.path.join(timestamp_dirpath,"actv_0-2.csv")) + data = parse_step_fn(os.path.join(self.timestamp_dirpath, "actv_0-2.csv")) result = { 'vp0:.input:micro0': { - 0: {'nans': 0.0,'norm': 5.550016}, - 1: {'nans': 0.0,'norm': 5.975112}, - 2: {'nans': 0.0,'norm': 5.789881} - }, + 0: {'nans': 0.0, 'norm': 5.550016}, + 1: {'nans': 0.0, 'norm': 5.975112}, + 2: {'nans': 0.0, 'norm': 5.789881} + }, 'vp0:.output:micro0': { - 0: {'nans': 0.0,'norm': 41.842655}, - 1: {'nans': 0.0,'norm': 44.40981}, - 2: {'nans': 0.0,'norm': 43.578354} - }, + 0: {'nans': 0.0, 'norm': 41.842655}, + 1: {'nans': 0.0, 'norm': 44.40981}, + 2: {'nans': 0.0, 'norm': 43.578354} + }, 'vp0:linear.input:micro0': { - 0: {'nans': 0.0,'norm': 5.550016}, - 1: {'nans': 0.0,'norm': 5.975112}, - 2: {'nans': 0.0,'norm': 5.789881} - }, + 0: {'nans': 0.0, 'norm': 5.550016}, + 1: {'nans': 0.0, 'norm': 5.975112}, + 2: {'nans': 0.0, 'norm': 5.789881} + }, 'vp0:linear.output:micro0': { - 0: {'nans': 0.0,'norm': 41.842655}, - 1: {'nans': 0.0,'norm': 44.40981}, - 2: {'nans': 0.0,'norm': 43.578354} - }, + 0: {'nans': 0.0, 'norm': 41.842655}, + 1: {'nans': 0.0, 'norm': 44.40981}, + 2: {'nans': 0.0, 'norm': 43.578354} + }, 'vp0:relu.input:micro0': { - 0: {'nans': 0.0,'norm': 41.842655}, - 1: {'nans': 0.0,'norm': 44.40981}, - 2: {'nans': 0.0,'norm': 43.578354} - }, + 0: {'nans': 0.0, 'norm': 41.842655}, + 1: {'nans': 0.0, 'norm': 44.40981}, + 2: {'nans': 0.0, 'norm': 43.578354} + }, 'vp0:relu.output:micro0': { - 0: {'nans': 0.0,'norm': 41.842655}, - 1: {'nans': 0.0,'norm': 44.40981}, - 2: {'nans': 0.0,'norm': 43.578354} - } + 0: {'nans': 0.0, 'norm': 41.842655}, + 1: {'nans': 0.0, 'norm': 44.40981}, + 2: {'nans': 0.0, 'norm': 43.578354} } - self.assertEqual(dict_equal(data, result), True) - tb_data = extract_scalars_from_tensorboard(os.path.join(csv2tb_dirpath, "actv")) + } + self.assertDictEqual(data, result) + tb_data = extract_scalars_from_tensorboard(os.path.join(self.csv2tb_dirpath, "actv")) print(tb_data) tb_result = { 'vp0:.input:micro0/nans': [(0, 0.0), - (1, 0.0), - (2, 0.0), - (3, 0.0), - (4, 0.0), - (5, 0.0), - (6, 0.0), - (7, 0.0), - (8, 0.0), - (9, 0.0)], + (1, 0.0), + (2, 0.0), + (3, 0.0), + (4, 0.0), + (5, 0.0), + (6, 0.0), + (7, 0.0), + (8, 0.0), + (9, 0.0)], 'vp0:.input:micro0/norm': [(0, 5.550015926361084), - (1, 5.975111961364746), - (2, 5.789881229400635), - (3, 6.052319049835205), - (4, 5.573315143585205), - (5, 5.864360809326172), - (6, 5.292460918426514), - (7, 5.477899074554443), - (8, 5.884613990783691), - (9, 5.456457138061523)], + (1, 5.975111961364746), + (2, 5.789881229400635), + (3, 6.052319049835205), + (4, 5.573315143585205), + (5, 5.864360809326172), + (6, 5.292460918426514), + (7, 5.477899074554443), + (8, 5.884613990783691), + (9, 5.456457138061523)], 'vp0:.output:micro0/nans': [(0, 0.0), - (1, 0.0), - (2, 0.0), - (3, 0.0), - (4, 0.0), - (5, 0.0), - (6, 0.0), - (7, 0.0), - (8, 0.0), - (9, 0.0)], - 'vp0:.output:micro0/norm': [(0, 41.842655181884766), - (1, 44.40980911254883), - (2, 43.57835388183594), - (3, 45.83631134033203), - (4, 42.0673828125), - (5, 43.46839141845703), - (6, 39.77947235107422), - (7, 40.200843811035156), - (8, 44.453147888183594), - (9, 40.841522216796875)], - 'vp0:linear.input:micro0/nans': [(0, 0.0), (1, 0.0), (2, 0.0), (3, 0.0), @@ -244,117 +219,136 @@ class TestGradMonitor(unittest.TestCase): (7, 0.0), (8, 0.0), (9, 0.0)], + 'vp0:.output:micro0/norm': [(0, 41.842655181884766), + (1, 44.40980911254883), + (2, 43.57835388183594), + (3, 45.83631134033203), + (4, 42.0673828125), + (5, 43.46839141845703), + (6, 39.77947235107422), + (7, 40.200843811035156), + (8, 44.453147888183594), + (9, 40.841522216796875)], + 'vp0:linear.input:micro0/nans': [(0, 0.0), + (1, 0.0), + (2, 0.0), + (3, 0.0), + (4, 0.0), + (5, 0.0), + (6, 0.0), + (7, 0.0), + (8, 0.0), + (9, 0.0)], 'vp0:linear.input:micro0/norm': [(0, 5.550015926361084), - (1, 5.975111961364746), - (2, 5.789881229400635), - (3, 6.052319049835205), - (4, 5.573315143585205), - (5, 5.864360809326172), - (6, 5.292460918426514), - (7, 5.477899074554443), - (8, 5.884613990783691), - (9, 5.456457138061523)], + (1, 5.975111961364746), + (2, 5.789881229400635), + (3, 6.052319049835205), + (4, 5.573315143585205), + (5, 5.864360809326172), + (6, 5.292460918426514), + (7, 5.477899074554443), + (8, 5.884613990783691), + (9, 5.456457138061523)], 'vp0:linear.output:micro0/nans': [(0, 0.0), - (1, 0.0), - (2, 0.0), - (3, 0.0), - (4, 0.0), - (5, 0.0), - (6, 0.0), - (7, 0.0), - (8, 0.0), - (9, 0.0)], + (1, 0.0), + (2, 0.0), + (3, 0.0), + (4, 0.0), + (5, 0.0), + (6, 0.0), + (7, 0.0), + (8, 0.0), + (9, 0.0)], 'vp0:linear.output:micro0/norm': [(0, 41.842655181884766), - (1, 44.40980911254883), - (2, 43.57835388183594), - (3, 45.83631134033203), - (4, 42.0673828125), - (5, 43.46839141845703), - (6, 39.77947235107422), - (7, 40.200843811035156), - (8, 44.453147888183594), - (9, 40.841522216796875)], + (1, 44.40980911254883), + (2, 43.57835388183594), + (3, 45.83631134033203), + (4, 42.0673828125), + (5, 43.46839141845703), + (6, 39.77947235107422), + (7, 40.200843811035156), + (8, 44.453147888183594), + (9, 40.841522216796875)], 'vp0:relu.input:micro0/nans': [(0, 0.0), - (1, 0.0), - (2, 0.0), - (3, 0.0), - (4, 0.0), - (5, 0.0), - (6, 0.0), - (7, 0.0), - (8, 0.0), - (9, 0.0)], + (1, 0.0), + (2, 0.0), + (3, 0.0), + (4, 0.0), + (5, 0.0), + (6, 0.0), + (7, 0.0), + (8, 0.0), + (9, 0.0)], 'vp0:relu.input:micro0/norm': [(0, 41.842655181884766), - (1, 44.40980911254883), - (2, 43.57835388183594), - (3, 45.83631134033203), - (4, 42.0673828125), - (5, 43.46839141845703), - (6, 39.77947235107422), - (7, 40.200843811035156), - (8, 44.453147888183594), - (9, 40.841522216796875)], + (1, 44.40980911254883), + (2, 43.57835388183594), + (3, 45.83631134033203), + (4, 42.0673828125), + (5, 43.46839141845703), + (6, 39.77947235107422), + (7, 40.200843811035156), + (8, 44.453147888183594), + (9, 40.841522216796875)], 'vp0:relu.output:micro0/nans': [(0, 0.0), - (1, 0.0), - (2, 0.0), - (3, 0.0), - (4, 0.0), - (5, 0.0), - (6, 0.0), - (7, 0.0), - (8, 0.0), - (9, 0.0)], + (1, 0.0), + (2, 0.0), + (3, 0.0), + (4, 0.0), + (5, 0.0), + (6, 0.0), + (7, 0.0), + (8, 0.0), + (9, 0.0)], 'vp0:relu.output:micro0/norm': [(0, 41.842655181884766), - (1, 44.40980911254883), - (2, 43.57835388183594), - (3, 45.83631134033203), - (4, 42.0673828125), - (5, 43.46839141845703), - (6, 39.77947235107422), - (7, 40.200843811035156), - (8, 44.453147888183594), - (9, 40.841522216796875)]} - self.assertEqual(compare_scalar_dicts(tb_data, tb_result), True) - + (1, 44.40980911254883), + (2, 43.57835388183594), + (3, 45.83631134033203), + (4, 42.0673828125), + (5, 43.46839141845703), + (6, 39.77947235107422), + (7, 40.200843811035156), + (8, 44.453147888183594), + (9, 40.841522216796875)]} + self.assertDictEqual(tb_data, tb_result) def test_actv_grad(self): - data = parse_step_fn(os.path.join(timestamp_dirpath,"actv_grad_0-2.csv")) + data = parse_step_fn(os.path.join(self.timestamp_dirpath, "actv_grad_0-2.csv")) nan = np.nan result = { 'vp0:.input:micro0': { - 0: {'norm': nan, 'nans': nan}, - 1: {'norm': nan, 'nans': nan}, + 0: {'norm': nan, 'nans': nan}, + 1: {'norm': nan, 'nans': nan}, 2: {'norm': nan, 'nans': nan} - }, + }, 'vp0:.output:micro0': { - 0: {'norm': 0.282843, 'nans': 0.0}, - 1: {'norm': 0.282617, 'nans': 0.0}, + 0: {'norm': 0.282843, 'nans': 0.0}, + 1: {'norm': 0.282617, 'nans': 0.0}, 2: {'norm': 0.282655, 'nans': 0.0} - }, + }, 'vp0:relu.input:micro0': { - 0: {'norm': 0.282843, 'nans': 0.0}, - 1: {'norm': 0.282617, 'nans': 0.0}, + 0: {'norm': 0.282843, 'nans': 0.0}, + 1: {'norm': 0.282617, 'nans': 0.0}, 2: {'norm': 0.282655, 'nans': 0.0} - }, + }, 'vp0:relu.output:micro0': { - 0: {'norm': 0.282843, 'nans': 0.0}, - 1: {'norm': 0.282617, 'nans': 0.0}, + 0: {'norm': 0.282843, 'nans': 0.0}, + 1: {'norm': 0.282617, 'nans': 0.0}, 2: {'norm': 0.282655, 'nans': 0.0} - }, + }, 'vp0:linear.input:micro0': { - 0: {'norm': nan, 'nans': nan}, - 1: {'norm': nan, 'nans': nan}, + 0: {'norm': nan, 'nans': nan}, + 1: {'norm': nan, 'nans': nan}, 2: {'norm': nan, 'nans': nan} - }, + }, 'vp0:linear.output:micro0': { - 0: {'norm': 0.282843, 'nans': 0.0}, - 1: {'norm': 0.282617, 'nans': 0.0}, + 0: {'norm': 0.282843, 'nans': 0.0}, + 1: {'norm': 0.282617, 'nans': 0.0}, 2: {'norm': 0.282655, 'nans': 0.0} - } } - self.assertEqual(dict_equal(data, result), True) - - tb_data = extract_scalars_from_tensorboard(os.path.join(csv2tb_dirpath, "actv_grad")) + } + print(data) + + tb_data = extract_scalars_from_tensorboard(os.path.join(self.csv2tb_dirpath, "actv_grad")) tb_result = { 'vp0:.input:micro0/nans': [(0, nan), (1, nan), @@ -475,88 +469,90 @@ class TestGradMonitor(unittest.TestCase): (6, 0.28316599130630493), (7, 0.28274500370025635), (8, 0.2833530008792877), - (9, 0.2825529873371124)]} - self.assertEqual(compare_scalar_dicts(tb_data, tb_result), True) + (9, 0.2825529873371124)] + } + print(tb_data) - def test_param(self): - data = parse_step_fn(os.path.join(timestamp_dirpath,"param_0-2.csv")) + data = parse_step_fn(os.path.join(self.timestamp_dirpath, "param_origin_0-2.csv")) result = { 'vp0:linear.bias': { 0: {'nans': 0.0, 'norm': 2.236068}, 1: {'nans': 0.0, 'norm': 2.236198}, 2: {'nans': 0.0, 'norm': 2.235769} - }, + }, 'vp0:linear.weight': { 0: {'nans': 0.0, 'norm': 7.071068}, 1: {'nans': 0.0, 'norm': 7.068808}, 2: {'nans': 0.0, 'norm': 7.06771} - } } - self.assertEqual(dict_equal(data, result), True) - tb_data = extract_scalars_from_tensorboard(os.path.join(csv2tb_dirpath, "param")) + } + self.assertDictEqual(data, result) + tb_data = extract_scalars_from_tensorboard(os.path.join(self.csv2tb_dirpath, "param_origin")) tb_result = { 'vp0:linear.weight/norm': [ - (0, 7.071067810058594), - (1, 7.068808078765869), - (2, 7.067709922790527), - (3, 7.0673418045043945), - (4, 7.066926956176758), - (5, 7.066311836242676), - (6, 7.065629959106445), - (7, 7.065262794494629), - (8, 7.065001964569092), - (9, 7.064840793609619)], + (0, 7.071067810058594), + (1, 7.068808078765869), + (2, 7.067709922790527), + (3, 7.0673418045043945), + (4, 7.066926956176758), + (5, 7.066311836242676), + (6, 7.065629959106445), + (7, 7.065262794494629), + (8, 7.065001964569092), + (9, 7.064840793609619)], 'vp0:linear.weight/nans': [ - (0, 0.0), - (1, 0.0), - (2, 0.0), - (3, 0.0), - (4, 0.0), - (5, 0.0), - (6, 0.0), - (7, 0.0), - (8, 0.0), - (9, 0.0)], + (0, 0.0), + (1, 0.0), + (2, 0.0), + (3, 0.0), + (4, 0.0), + (5, 0.0), + (6, 0.0), + (7, 0.0), + (8, 0.0), + (9, 0.0)], 'vp0:linear.bias/norm': [ - (0, 2.2360680103302), - (1, 2.2361979484558105), - (2, 2.235769033432007), - (3, 2.235903024673462), - (4, 2.2360129356384277), - (5, 2.2359039783477783), - (6, 2.2357990741729736), - (7, 2.2357349395751953), - (8, 2.2356700897216797), - (9, 2.235619068145752)], + (0, 2.2360680103302), + (1, 2.2361979484558105), + (2, 2.235769033432007), + (3, 2.235903024673462), + (4, 2.2360129356384277), + (5, 2.2359039783477783), + (6, 2.2357990741729736), + (7, 2.2357349395751953), + (8, 2.2356700897216797), + (9, 2.235619068145752) + ], 'vp0:linear.bias/nans': [ - (0, 0.0), - (1, 0.0), - (2, 0.0), - (3, 0.0), - (4, 0.0), - (5, 0.0), - (6, 0.0), - (7, 0.0), - (8, 0.0), - (9, 0.0)] - } - self.assertEqual(compare_scalar_dicts(tb_data, tb_result), True) + (0, 0.0), + (1, 0.0), + (2, 0.0), + (3, 0.0), + (4, 0.0), + (5, 0.0), + (6, 0.0), + (7, 0.0), + (8, 0.0), + (9, 0.0) + ] + } + self.assertDictEqual(tb_data, tb_result) def test_exp_avg(self): - data = parse_step_fn(os.path.join(timestamp_dirpath,"exp_avg_0-2.csv")) + data = parse_step_fn(os.path.join(self.timestamp_dirpath, "exp_avg_0-2.csv")) result = { 'vp0:linear.bias': { 1: {'nans': 0.0, 'norm': 0.024495}, 2: {'nans': 0.0, 'norm': 0.052203} - }, + }, 'vp0:linear.weight': { 1: {'nans': 0.0, 'norm': 0.052394}, 2: {'nans': 0.0, 'norm': 0.099221} - } } - self.assertEqual(dict_equal(data, result), True) - tb_data = extract_scalars_from_tensorboard(os.path.join(csv2tb_dirpath, "exp_avg")) + } + self.assertDictEqual(data, result) + tb_data = extract_scalars_from_tensorboard(os.path.join(self.csv2tb_dirpath, "exp_avg")) tb_result = { 'vp0:linear.bias/nans': [(1, 0.0), (2, 0.0), @@ -594,22 +590,22 @@ class TestGradMonitor(unittest.TestCase): (7, 0.11372199654579163), (8, 0.12264800071716309), (9, 0.09017200022935867)]} - self.assertEqual(compare_scalar_dicts(tb_data, tb_result), True) + self.assertDictEqual(tb_data, tb_result) def test_exp_avg_sq(self): - data = parse_step_fn(os.path.join(timestamp_dirpath,"exp_avg_sq_0-2.csv")) + data = parse_step_fn(os.path.join(self.timestamp_dirpath, "exp_avg_sq_0-2.csv")) result = { 'vp0:linear.bias': { 1: {'nans': 0.0, 'norm': 4.2e-05}, 2: {'nans': 0.0, 'norm': 9.6e-05} - }, + }, 'vp0:linear.weight': { 1: {'nans': 0.0, 'norm': 6.7e-05}, 2: {'nans': 0.0, 'norm': 0.000126} - } } - self.assertEqual(dict_equal(data, result), True) - tb_data = extract_scalars_from_tensorboard(os.path.join(csv2tb_dirpath, "exp_avg_sq")) + } + self.assertDictEqual(data, result) + tb_data = extract_scalars_from_tensorboard(os.path.join(self.csv2tb_dirpath, "exp_avg_sq")) tb_result = { 'vp0:linear.bias/nans': [(1, 0.0), (2, 0.0), @@ -647,24 +643,24 @@ class TestGradMonitor(unittest.TestCase): (7, 0.00026000000070780516), (8, 0.00028700000257231295), (9, 0.0003060000017285347)]} - self.assertEqual(compare_scalar_dicts(tb_data, tb_result), True) - + self.assertDictEqual(tb_data, tb_result) + def test_grad_reduced(self): - data = parse_step_fn(os.path.join(timestamp_dirpath,"grad_reduced_0-2.csv")) + data = parse_step_fn(os.path.join(self.timestamp_dirpath, "grad_reduced_0-2.csv")) result = { 'vp0:linear.bias': { 0: {'nans': 0.0, 'norm': 0.244949}, 1: {'nans': 0.0, 'norm': 0.314345}, 2: {'nans': 0.0, 'norm': 0.281475} - }, + }, 'vp0:linear.weight': { 0: {'nans': 0.0, 'norm': 0.523935}, 1: {'nans': 0.0, 'norm': 0.595672}, 2: {'nans': 0.0, 'norm': 0.497603} - } } - self.assertEqual(dict_equal(data, result), True) - tb_data = extract_scalars_from_tensorboard(os.path.join(csv2tb_dirpath, "grad_reduced")) + } + self.assertDictEqual(data, result) + tb_data = extract_scalars_from_tensorboard(os.path.join(self.csv2tb_dirpath, "grad_reduced")) tb_result = { 'vp0:linear.bias/nans': [(0, 0.0), (1, 0.0), @@ -706,25 +702,25 @@ class TestGradMonitor(unittest.TestCase): (7, 0.4831080138683319), (8, 0.3234719932079315), (9, 0.32385098934173584)]} - self.assertEqual(compare_scalar_dicts(tb_data, tb_result), True) - + self.assertDictEqual(tb_data, tb_result) + def test_grad_unreduced(self): - data = parse_step_fn(os.path.join(timestamp_dirpath,"grad_unreduced_0-2.csv")) + data = parse_step_fn(os.path.join(self.timestamp_dirpath, "grad_unreduced_0-2.csv")) result = { 'vp0:linear.bias': { 0: {'nans': 0.0, 'norm': 0.244949}, 1: {'nans': 0.0, 'norm': 0.314345}, 2: {'nans': 0.0, 'norm': 0.281475} - }, + }, 'vp0:linear.weight': { 0: {'nans': 0.0, 'norm': 0.523935}, 1: {'nans': 0.0, 'norm': 0.595672}, 2: {'nans': 0.0, 'norm': 0.497603} - } } - self.assertEqual(dict_equal(data, result), True) + } + self.assertDictEqual(data, result) - tb_data = extract_scalars_from_tensorboard(os.path.join(csv2tb_dirpath, "grad_unreduced")) + tb_data = extract_scalars_from_tensorboard(os.path.join(self.csv2tb_dirpath, "grad_unreduced")) tb_result = { 'vp0:linear.bias/nans': [(0, 0.0), (1, 0.0), @@ -766,4 +762,8 @@ class TestGradMonitor(unittest.TestCase): (7, 0.4831080138683319), (8, 0.3234719932079315), (9, 0.32385098934173584)]} - self.assertEqual(compare_scalar_dicts(tb_data, tb_result), True) + self.assertDictEqual(tb_data, tb_result) + + +if __name__ == '__main__': + unittest.main() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_module_hook.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_module_hook.py index 66d016f9487a4e7f7fc747dfb021b1f887c51f4a..6c3d2b925a4fbcd7ec7c81b85614b6be0e731b0c 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_module_hook.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_module_hook.py @@ -90,13 +90,13 @@ class TestModuleHook(unittest.TestCase): self.assertTrue(os.path.exists(actv_grad_0_csv)) # validate columns and lines actv_0 = pd.read_csv(actv_0_csv) - expect_columns = ['vpp_stage', 'name', 'step', 'micro_step', 'norm', 'nans'] + expect_columns = ['vpp_stage', 'name', 'step', 'micro_step', 'norm', 'nans', "shape", "dtype"] self.assertListEqual(list(actv_0.columns), expect_columns) - self.assertEqual(actv_0.shape, tuple([6, 6])) + self.assertEqual(actv_0.shape, tuple([6, 8])) actv_grad_0 = pd.read_csv(actv_grad_0_csv) - expect_columns = ['vpp_stage', 'name', 'step', 'micro_step', 'norm', 'nans'] + expect_columns = ['vpp_stage', 'name', 'step', 'micro_step', 'norm', 'nans', "shape", "dtype"] self.assertListEqual(list(actv_grad_0.columns), expect_columns) - self.assertEqual(actv_0.shape, tuple([6, 6])) + self.assertEqual(actv_0.shape, tuple([6, 8])) def test_wg_distribution(self): self.get_dist_mock(False) @@ -113,13 +113,13 @@ class TestModuleHook(unittest.TestCase): self.assertTrue(os.path.exists(grad_reduced_0_csv)) self.assertTrue(os.path.exists(grad_unreduced_0_csv)) # validate columns and lines - expect_columns = ["vpp_stage", "name", "step", "norm"] + expect_columns = ["vpp_stage", "name", "step", "norm", "shape", "dtype"] grad_reduced_0 = pd.read_csv(grad_reduced_0_csv) self.assertListEqual(list(grad_reduced_0.columns), expect_columns) - self.assertEqual(grad_reduced_0.shape, tuple([2, 4])) + self.assertEqual(grad_reduced_0.shape, tuple([2, 6])) grad_unreduced_0 = pd.read_csv(grad_unreduced_0_csv) self.assertListEqual(list(grad_unreduced_0.columns), expect_columns) - self.assertEqual(grad_unreduced_0.shape, tuple([2, 4])) + self.assertEqual(grad_unreduced_0.shape, tuple([2, 6])) def test_mv_distribution(self): self.get_dist_mock(False) @@ -136,13 +136,13 @@ class TestModuleHook(unittest.TestCase): self.assertTrue(os.path.exists(exp_avg_1_csv)) self.assertTrue(os.path.exists(exp_avg_sq_1_csv)) # validate columns and lines - expect_columns = ["vpp_stage", "name", "step", "norm"] + expect_columns = ["vpp_stage", "name", "step", "norm", "shape", "dtype"] exp_avg_1 = pd.read_csv(exp_avg_1_csv) self.assertListEqual(list(exp_avg_1.columns), expect_columns) - self.assertEqual(exp_avg_1.shape, tuple([2, 4])) + self.assertEqual(exp_avg_1.shape, tuple([2, 6])) exp_avg_sq_1 = pd.read_csv(exp_avg_sq_1_csv) self.assertListEqual(list(exp_avg_sq_1.columns), expect_columns) - self.assertEqual(exp_avg_sq_1.shape, tuple([2, 4])) + self.assertEqual(exp_avg_sq_1.shape, tuple([2, 6])) def test_ur_distribution(self): self.get_dist_mock(False) @@ -167,6 +167,18 @@ class TestModuleHook(unittest.TestCase): ) self.assertIsNotNone(hooker) + def test_stack_collect(self): + self.get_dist_mock(False) + stack_monitor_output = "./test_stack_info" + clean_output(stack_monitor_output) + os.environ[MonitorConst.MONITOR_OUTPUT_DIR] = stack_monitor_output + stack_config = os.path.join(base_dir, "config/stack_config.json") + monitor_demo(stack_config) + output_dir_list = os.listdir(stack_monitor_output) + self.assertEqual(len(output_dir_list), 1) + stack_csv_path = os.path.join(stack_monitor_output, output_dir_list[0], "stack_info.csv") + self.assertTrue(os.path.exists(stack_csv_path)) + def test_adhoc_check(self): # mock dist self.get_dist_mock(True) @@ -261,61 +273,6 @@ class TestParamIsDataParallelDuplicate(unittest.TestCase): self.assertFalse(result) -class TestModuleHookContext(unittest.TestCase): - def setUp(self): - self.module_name = "test_module" - self.context = ModuleHookContext(self.module_name) - self.context.struct = { - Const.INPUT: { - "config": "tuple[1]", - "0": "size=(2, 784), dtype=torch.float32", - }, - Const.OUTPUT: { - "config": "tensor", - "tensor": "size=(2, 10), dtype=torch.float32" - }, - MonitorConst.INPUT_GRAD: { - "config": "tuple[1]", - "0": "size=(2, 784), dtype=torch.float32" - }, - MonitorConst.OUTPUT_GRAD: { - "config": "tuple[1]", - "0": "size=(2, 10), dtype=torch.float32" - } - } - self.target_config = { - self.module_name: { - Const.INPUT: "tuple[1]:0", - Const.OUTPUT: "tensor", - MonitorConst.INPUT_GRAD: "tuple[1]:0" - } - } - - def test_set_format_by_arg_module_name_in_target_config(self): - self.context.set_format_by_arg(Const.INPUT, self.target_config) - self.assertEqual(self.context.format_by_arg[Const.INPUT], "tuple[1]:0") - self.context.set_format_by_arg(Const.OUTPUT, self.target_config) - self.assertEqual(self.context.format_by_arg[Const.OUTPUT], "tensor") - self.context.set_format_by_arg(MonitorConst.INPUT_GRAD, self.target_config) - self.assertEqual(self.context.format_by_arg[MonitorConst.INPUT_GRAD], "tuple[1]:0") - self.context.set_format_by_arg(MonitorConst.OUTPUT_GRAD, self.target_config) - self.assertEqual(self.context.format_by_arg[MonitorConst.OUTPUT_GRAD], "tuple[1]") - - def test_set_format_by_arg_module_name_not_in_target_config(self): - target_config = {} - self.context.set_format_by_arg(Const.INPUT, target_config) - self.assertEqual(self.context.format_by_arg[Const.INPUT], "tuple[1]") - self.context.set_format_by_arg(Const.OUTPUT, target_config) - self.assertEqual(self.context.format_by_arg[Const.OUTPUT], "tensor") - - @patch('msprobe.pytorch.monitor.module_hook.logger') - def test_set_format_by_arg_target_module_config_error(self, mock_logger): - target_config = {self.module_name: {Const.INPUT: 123}} - self.context.set_format_by_arg(Const.INPUT, target_config) - self.assertIsNone(self.context.format_by_arg.get(Const.INPUT)) - mock_logger.warning_on_rank_0.assert_called_once() - - class TestContext(unittest.TestCase): def test_communication_context(self): cc_ctx = CommunicationContext() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_monitor_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_monitor_utils.py index 0462ac3f39531119b40d3cc5051fad77f687b9b5..87822ab0503bd21e0546d8c846d69f56204eb048 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_monitor_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_monitor_utils.py @@ -44,12 +44,12 @@ class TestValidationFunctions(unittest.TestCase): def test_validate_ops(self): ops = ['op1', 'op2', 'norm', 'max'] valid_ops = validate_ops(ops) - self.assertEqual(valid_ops, ['norm', 'max']) + self.assertEqual(valid_ops, ['norm', 'max', "shape", "dtype"]) def test_no_valid_ops(self): ops = ['op1', 'op2'] valid_ops = validate_ops(ops) - target_ops = [MonitorConst.OP_LIST[0]] + target_ops = [MonitorConst.OP_LIST[0], "shape", "dtype"] self.assertEqual(valid_ops, target_ops) def test_validate_ranks(self): @@ -104,7 +104,7 @@ class TestValidationFunctions(unittest.TestCase): 'alert': {'rules': [{'rule_name': 'AnomalyTurbulence', 'args': {'threshold': 10.0}}], 'dump': True} } validate_config(config) - target_ops = [MonitorConst.OP_LIST[0]] + target_ops = [MonitorConst.OP_LIST[0], "shape", "dtype"] self.assertEqual(config["ops"], target_ops) del config["targets"] validate_config(config) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_optimizer_collect.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_optimizer_collect.py index 793b086b02db03f8a04b159f35f1df55fc1a9d2c..242f70e50e4cdc2b2b50dc99be627bdec47ad263 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_optimizer_collect.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/monitor/test_optimizer_collect.py @@ -3,18 +3,51 @@ from collections import defaultdict from unittest.mock import Mock, patch, MagicMock import torch +from msprobe.core.common.const import MonitorConst from msprobe.pytorch.monitor.optimizer_collect import OptimizerMon, \ - OptimizerMonFactory, DummyOptimizerMon, \ - MixPrecisionOptimizerMon, MegatronDistributedOptimizerMon, MegatronFP32OptimizerMon, \ + OptimizerMonFactory, MixPrecisionOptimizerMon, MegatronDistributedOptimizerMon, \ MegatronChainedDistributedOptimizerMon, MegatronChainedMixPrecisionOptimizerMon, \ - DeepSpeedZeroOptimizerStage0Mon, DeepSpeedZeroOptimizerStage1or2Mon, DeepSpeedZeroOptimizerStage3Mon - -from msprobe.pytorch.monitor.utils import MVResult, MVGradResult - + DeepSpeedZeroOptimizerMon, DeepSpeedZeroOptimizerStage0Mon, \ + DeepSpeedZeroOptimizerStage1or2Mon, DeepSpeedZeroOptimizerStage3Mon +from msprobe.pytorch.monitor.utils import MVResult + + +def setup_param_groups(num_groups=2, params_per_group=5): + bit16_groups = [] + param_names = {} + grad_position = {} + param_slice_mappings = [] + count = 0 + for group_idx in range(num_groups): + group = [] + param_slice_mapping = {} + offset = 0 + for i in range(params_per_group): + name = f'param{group_idx}_{i}' + p = torch.nn.Parameter(torch.randn(2,3, dtype=torch.bfloat16)) + p.ds_tensor = torch.nn.Parameter(torch.randn(1,3, dtype=torch.bfloat16)) + p.ds_id = count + param_slice_mapping[name] = MagicMock(start=offset, numel=p.numel()) + group.append(p) + param_names[p] = name + grad_position[count] = [group_idx, offset, p.numel()] + offset += p.numel() + count += 1 + bit16_groups.append(group) + param_slice_mappings.append(param_slice_mapping) + + return bit16_groups, param_names, param_slice_mappings, grad_position + +def setup_mock_monitor(): + mock_monitor = MagicMock() + mock_monitor.mv_distribution = True + mock_monitor.mg_direction = False + mock_monitor.ur_distribution = False + + return mock_monitor class TestOptimizerMon(unittest.TestCase): def setUp(self) -> None: - # 初始化需要的monitor, torch_opt, params2name等对象 self.monitor = Mock() self.monitor.mv_distribution = True self.monitor.mg_direction = True @@ -23,11 +56,11 @@ class TestOptimizerMon(unittest.TestCase): self.monitor.ratio_heatmap_visualizer = {'param1': Mock(), 'param2': Mock()} def test_fetch_mv(self): - optimizer_mon = OptimizerMon() - res = optimizer_mon.fetch_mv(None, None, None) - self.assertEqual(res, None) + optimizer_mon = OptimizerMon(None) + res = optimizer_mon.fetch_mv(None, {}) + self.assertEqual(res.exp_avg, {}) - def test_fetch_mv_in_adam(self): + def test_fetch_mv(self): self.torch_opt = Mock() self.torch_opt.state = { 'param1': {'exp_avg': torch.tensor(0.1), 'exp_avg_sq': torch.tensor(0.2), 'step': torch.tensor(10)}, @@ -37,48 +70,10 @@ class TestOptimizerMon(unittest.TestCase): self.torch_opt.defaults = {'betas': (0.9, 0.999), 'eps': 1e-8} self.params2name = {'param1': 'param1', 'param2': 'param2'} - self.optimizer_mon = OptimizerMon() - result = self.optimizer_mon._fetch_mv_in_adam(self.monitor, self.torch_opt, self.params2name) + self.optimizer_mon = OptimizerMon(None) + result = self.optimizer_mon.fetch_mv(self.monitor, self.params2name) self.assertIsInstance(result, MVResult) - @patch('msprobe.pytorch.monitor.optimizer_collect.dist') - def test_fetch_mv_grad_in_adam(self, mock_dist): - self.optimizer_mon = OptimizerMon() - self.monitor = MagicMock() - self.torch_opt = MagicMock() - self.params2name = defaultdict(str) - self.name2indices = defaultdict(tuple) - self.fp32_partitioned_groups_flat = defaultdict(torch.Tensor) - - # Mocking the dist.get_rank() and dist.get_world_size() - mock_dist.get_rank.return_value = 0 - mock_dist.get_world_size.return_value = 1 - - # Mocking the wrapped_optimizer - self.torch_opt.state = defaultdict(dict) - self.torch_opt.averaged_gradients = defaultdict(torch.Tensor) - self.torch_opt.partition_size = defaultdict(int) - self.torch_opt.flatten_dense_tensors_aligned = MagicMock() - self.torch_opt.flatten = MagicMock() - - # Mocking the torch_opt.param_groups - self.torch_opt.param_groups = [{'step': 1, 'betas': (0.9, 0.999)}, - {'step': 2, 'betas': (0.9, 0.999)}, - {'step': 3, 'betas': (0.9, 0.999)}] - - # Mocking the monitor.mv_distribution, monitor.mg_direction, monitor.ur_distribution - self.monitor.mv_distribution = True - self.monitor.mg_direction = True - self.monitor.ur_distribution = True - - # Mocking the monitor.update_heatmap_visualizer and monitor.ratio_heatmap_visualizer - self.monitor.update_heatmap_visualizer = defaultdict(MagicMock) - self.monitor.ratio_heatmap_visualizer = defaultdict(MagicMock) - - result = self.optimizer_mon._fetch_mv_grad_in_adam(self.monitor, self.torch_opt, self.params2name, - self.name2indices, self.fp32_partitioned_groups_flat) - self.assertIsInstance(result, MVGradResult) - class TestMixPrecisionOptimizerMon(unittest.TestCase): def test_fetch_mv_with_fp16_to_fp32_param_and_mix_prec_opt(self): @@ -89,16 +84,16 @@ class TestMixPrecisionOptimizerMon(unittest.TestCase): self.mix_prec_opt = MagicMock() self.mix_prec_opt.float16_groups = [MagicMock()] self.mix_prec_opt.fp32_from_float16_groups = [MagicMock()] - self.optimizer = MixPrecisionOptimizerMon() + self.optimizer = MixPrecisionOptimizerMon(self.torch_opt) self.optimizer.fp16_to_fp32_param = {} - # Mock _fetch_mv_in_adam method and set a fixed return value + # Mock fetch_mv method and set a fixed return value mv_result = MVResult(exp_avg={}, exp_avg_sq={}, update={}, ratio={}) - self.mock_fetch_mv_in_adam = MagicMock(return_value=mv_result) - self.optimizer._fetch_mv_in_adam = self.mock_fetch_mv_in_adam + self.mock_fetch_mv = MagicMock(return_value=mv_result) + self.optimizer.fetch_mv = self.mock_fetch_mv - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) - self.mock_fetch_mv_in_adam.assert_called_once_with(self.monitor, self.torch_opt, self.params2name) + res = self.optimizer.fetch_mv(self.monitor, self.params2name) + self.mock_fetch_mv.assert_called_once_with(self.monitor, self.params2name) self.assertIsInstance(res, MVResult) @@ -110,17 +105,17 @@ class TestChainedMixPrecisionOptimizerMon(unittest.TestCase): self.params2name = MagicMock() self.torch_opt.float16_groups = [MagicMock()] self.torch_opt.fp32_from_float16_groups = [MagicMock()] - self.optimizer = MegatronChainedMixPrecisionOptimizerMon() + self.optimizer = MegatronChainedMixPrecisionOptimizerMon(self.torch_opt) self.optimizer.optimizer = [MagicMock(), MagicMock()] self.optimizer.fp16_to_fp32_param = {} - # Mock _fetch_mv_in_adam method and set a fixed return value + # Mock fetch_mv method and set a fixed return value mv_result = MVResult(exp_avg={}, exp_avg_sq={}, update={}, ratio={}) - self.mock_fetch_mv_in_adam = MagicMock(return_value=mv_result) - self.optimizer._fetch_mv_in_adam = self.mock_fetch_mv_in_adam + self.mock_fetch_mv = MagicMock(return_value=mv_result) + self.optimizer.fetch_mv = self.mock_fetch_mv - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) - self.mock_fetch_mv_in_adam.assert_called_once_with(self.monitor, self.torch_opt, self.params2name) + res = self.optimizer.fetch_mv(self.monitor, self.params2name) + self.mock_fetch_mv.assert_called_once_with(self.monitor, self.params2name) self.assertIsInstance(res, MVResult) @@ -129,26 +124,27 @@ class TestMegatronChainedDistributedOptimizerMon(unittest.TestCase): self.monitor = MagicMock() self.torch_opt = MagicMock() self.params2name = MagicMock() + self.torch_opt.chained_optimizers = [MagicMock(), MagicMock()] mv_result = MVResult(exp_avg={}, exp_avg_sq={}, update={}, ratio={}) - self.mock_fetch_mv_in_adam = MagicMock(return_value=mv_result) - self.optimizer = MegatronChainedDistributedOptimizerMon() + self.mock_fetch_mv = MagicMock(return_value=mv_result) + self.optimizer = MegatronChainedDistributedOptimizerMon(self.torch_opt) def test_fetch_mv_with_valid_optimizer(self): - self.torch_opt.model_float16_groups = [MagicMock()] - self.torch_opt.shard_fp32_from_float16_groups = [MagicMock()] - self.optimizer._fetch_mv_in_adam = self.mock_fetch_mv_in_adam + for opt in self.torch_opt.chained_optimizers: + opt.model_float16_groups = [MagicMock()] + opt.shard_fp32_from_float16_groups = [MagicMock()] + self.optimizer.fetch_mv = self.mock_fetch_mv - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) + res = self.optimizer.fetch_mv(self.monitor, self.params2name) self.assertIsInstance(res, MVResult) def test_fetch_mv_with_invalid_optimizer(self): - self.torch_opt = Mock() - self.torch_opt.model_float16_groups = None - self.torch_opt.shard_fp32_from_float16_groups = None - self.optimizer._fetch_mv_in_adam = self.mock_fetch_mv_in_adam + for opt in self.torch_opt.chained_optimizers: + del opt.model_float16_groups + del opt.shard_fp32_from_float16_groups with self.assertRaises(Exception): - self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) + self.optimizer.fetch_mv(self.monitor, self.params2name) class TestMegatronDistributedOptimizerMon(unittest.TestCase): @@ -157,25 +153,23 @@ class TestMegatronDistributedOptimizerMon(unittest.TestCase): self.torch_opt = MagicMock() self.params2name = MagicMock() mv_result = MVResult(exp_avg={}, exp_avg_sq={}, update={}, ratio={}) - self.mock_fetch_mv_in_adam = MagicMock(return_value=mv_result) - self.optimizer = MegatronDistributedOptimizerMon() + self.mock_fetch_mv = MagicMock(return_value=mv_result) + self.optimizer = MegatronDistributedOptimizerMon(self.torch_opt) def test_fetch_mv_with_valid_optimizer(self): self.torch_opt.model_float16_groups = [MagicMock()] self.torch_opt.shard_fp32_from_float16_groups = [MagicMock()] - self.optimizer._fetch_mv_in_adam = self.mock_fetch_mv_in_adam + self.optimizer.fetch_mv = self.mock_fetch_mv - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) + res = self.optimizer.fetch_mv(self.monitor, self.params2name) self.assertIsInstance(res, MVResult) def test_fetch_mv_with_invalid_optimizer(self): - self.torch_opt = Mock() self.torch_opt.model_float16_groups = None self.torch_opt.shard_fp32_from_float16_groups = None - self.optimizer._fetch_mv_in_adam = self.mock_fetch_mv_in_adam with self.assertRaises(Exception): - self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) + self.optimizer.fetch_mv(self.monitor, self.params2name) class TestCommonFetchMv(unittest.TestCase): @@ -184,103 +178,183 @@ class TestCommonFetchMv(unittest.TestCase): self.torch_opt = MagicMock() self.params2name = MagicMock() - def test_megatron_fp32_optimizer_mon(self): - self.optimizer = MegatronFP32OptimizerMon() - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) + def test_optimizer_mon(self): + self.optimizer = OptimizerMon(None) + res = self.optimizer.fetch_mv(self.monitor, self.params2name) self.assertIsInstance(res, MVResult) - def test_deepspeed_zero_optimizer_stage0_mon(self): - self.optimizer = DeepSpeedZeroOptimizerStage0Mon() - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) - self.assertIsInstance(res, MVResult) - def test_dummy_optimizer_mon(self): - self.optimizer = DummyOptimizerMon() - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) - self.assertIsInstance(res, MVResult) +class TestDeepSpeedZeroOptimizer(unittest.TestCase): + def setUp(self): + bit16_groups, param_names, param_slice_mappings, _ = setup_param_groups() + mock_opt = MagicMock() + mock_opt.state_dict.return_value = { + 'param_slice_mappings': param_slice_mappings + } + mock_opt.param_names = param_names + mock_opt.bit16_groups = bit16_groups + self.torch_opt = mock_opt + self.mock_monitor = setup_mock_monitor() + self.optimizer_mon = DeepSpeedZeroOptimizerMon(mock_opt) + self.optimizer_mon.bit16_groups = mock_opt.bit16_groups + self.optimizer_mon.param2group = self.optimizer_mon.get_group_index() + + def test_param_not_in_partition(self): + param_in_partition = list(self.torch_opt.param_names.keys())[0] + param_not_in_partition = torch.randn(2,3) + + self.assertFalse( + self.optimizer_mon.param_not_in_partition(param_in_partition, 0) + ) + self.assertTrue( + self.optimizer_mon.param_not_in_partition(param_not_in_partition, 0) + ) + + def test_get_position(self): + param_in_partition = list(self.torch_opt.param_names.keys())[0] + start, numel = self.optimizer_mon.get_position(param_in_partition, 0) + self.assertEqual(start, 0) + self.assertEqual(numel, 6) -class TestDeepSpeedZeroOptimizerStage3Mon(unittest.TestCase): - def test_get_param_index(self): - self.torch_opt = Mock() - self.torch_opt.fp16_partitioned_groups = [ - [Mock(flatten=lambda: [1, 2, 3]), - Mock(flatten=lambda: [4, 5])], - [Mock(flatten=lambda: [6, 7, 8, 9])] - ] - self.params2name = {'param1': 'weight1', 'param2': 'weight2'} - self.name2index = {'weight1': 0, 'weight2': 2} + def test_get_group_index(self): + param = list(self.torch_opt.param_names.keys())[6] + self.assertEqual(self.optimizer_mon.param2group[param], 1) - optimizer_stage3_mon = DeepSpeedZeroOptimizerStage3Mon() - name2indices = optimizer_stage3_mon.get_param_index(self.params2name, self.name2index, self.torch_opt) +class TestDeepSpeedZeroOptimizerStage0Mon(unittest.TestCase): + def setUp(self): + bit16_groups, param_names, param_slice_mappings, _ = setup_param_groups() - expected_name2indices = {'weight1': (0, 3, 0, None), 'weight2': (5, 9, 1, None)} - self.assertDictEqual(dict(name2indices), expected_name2indices) + mock_opt = MagicMock() + mock_opt.state_dict.return_value = { + 'param_slice_mappings': param_slice_mappings + } + mock_opt.param_names = param_names + mock_opt.bf16_groups = bit16_groups + mock_opt.fp32_groups_flat_partition = [torch.stack(group,dim=0).flatten().float() \ + for group in bit16_groups]# mock name 2 index in subgroup + mock_opt.state = { + flat_group: { + 'exp_avg': torch.ones_like(flat_group), + 'exp_avg_sq': torch.ones_like(flat_group) + } for flat_group in mock_opt.fp32_groups_flat_partition + } + mock_opt.cpu_offload = False + + self.torch_opt = mock_opt + self.mock_monitor = setup_mock_monitor() + self.optimizer_mon = DeepSpeedZeroOptimizerStage0Mon(mock_opt) + + def test_get_grad_for_param(self): + param = list(self.torch_opt.param_names.keys())[0] + group_idx = 0 + param_id = 2 + grad_expected = torch.randn_like(param) + self.torch_opt.fp32_groups_gradient_dict = [[0, 0, grad_expected, 0]] + grad = self.optimizer_mon.get_grad_for_param(param, group_idx, param_id) + + self.assertTrue(torch.equal(grad_expected, grad)) + + def test_fetch_grad(self): + self.torch_opt.fp32_groups_gradient_dict = [[torch.randn_like(param) for param in group] for group in self.optimizer_mon.bit16_groups] + self.mock_monitor.name2tag = {name:{MonitorConst.POST_GRAD: name} for name in self.torch_opt.param_names.values()} + result = self.optimizer_mon.fetch_grad(self.mock_monitor, self.torch_opt.param_names) + for _, name in self.torch_opt.param_names.items(): + group_index, param_id = [int(i) for i in name.replace('param','').split('_')] + self.assertTrue(torch.equal(result[name], self.torch_opt.fp32_groups_gradient_dict[group_index][param_id])) def test_fetch_mv(self): - self.monitor = MagicMock() - self.torch_opt = MagicMock() - self.params2name = MagicMock() - self.torch_opt.fp16_partitioned_groups = MagicMock() - self.optimizer = DeepSpeedZeroOptimizerStage3Mon() - - # mock _fetch_mv_grad_in_adam - mv_result = MVGradResult(exp_avg={}, exp_avg_sq={}, update={}, ratio={}, grad={}) - self.mock_fetch_mv_grad_in_adam = MagicMock(return_value=mv_result) - self.optimizer._fetch_mv_grad_in_adam = self.mock_fetch_mv_grad_in_adam - - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) - self.assertIsInstance(res, MVGradResult) + result = self.optimizer_mon.fetch_mv(self.mock_monitor, self.torch_opt.param_names) + for param, name in self.torch_opt.param_names.items(): + self.assertTrue(torch.equal(result.exp_avg[name], torch.ones_like(param).flatten())) + self.assertTrue(torch.equal(result.exp_avg_sq[name], torch.ones_like(param).flatten())) class TestDeepSpeedZeroOptimizerStage1or2Mon(unittest.TestCase): - def test_get_group_index(self): - self.fp32_length = [10, 20, 30, 40] - self.world_size = 4 - self.indexes = [5, 7, 12, 25, 35, 45] - self.expected_results = [(40, 0), (40, 0), (12, 1), (24, 2), (34, 2), (40, 0)] - - optimizer = DeepSpeedZeroOptimizerStage1or2Mon() - results = [optimizer.get_group_index(self.fp32_length, self.world_size, index) for index in self.indexes] - self.assertEqual(results, self.expected_results) + def setUp(self): + bit16_groups, param_names, param_slice_mappings, _ = setup_param_groups() - @patch('msprobe.pytorch.monitor.optimizer_collect.dist') - def test_get_param_index(self, mock_dist): - mock_dist.get_world_size.return_value = 4 + mock_opt = MagicMock() + mock_opt.state_dict.return_value = { + 'param_slice_mappings': param_slice_mappings + } + mock_opt.param_names = param_names + mock_opt.bit16_groups = bit16_groups + mock_opt.single_partition_of_fp32_groups = [torch.stack(group,dim=0).flatten().float() \ + for group in bit16_groups] + mock_opt.averaged_gradients = {group_idx: [torch.randn_like(param) for param in group] for group_idx, group in enumerate(bit16_groups)}# mock name 2 index in subgroup + mock_opt.state = { + flat_group: { + 'exp_avg': torch.ones_like(flat_group), + 'exp_avg_sq': torch.ones_like(flat_group) + } for flat_group in mock_opt.single_partition_of_fp32_groups + } + mock_opt.cpu_offload = False + + self.torch_opt = mock_opt + self.mock_monitor = setup_mock_monitor() + self.optimizer_mon = DeepSpeedZeroOptimizerStage1or2Mon(mock_opt) + + def test_get_grad_for_param(self): + param = list(self.torch_opt.param_names.keys())[0] + group_idx = 0 + param_id = 2 + grad_expected = torch.randn_like(param) + self.torch_opt.averaged_gradients = [[0, 0, grad_expected, 0]] + grad = self.optimizer_mon.get_grad_for_param(param, group_idx, param_id) + + self.assertTrue(torch.equal(grad_expected, grad)) + + def test_fetch_grad(self): + self.mock_monitor.name2tag = {name:{MonitorConst.POST_GRAD: name} for name in self.torch_opt.param_names.values()} + result = self.optimizer_mon.fetch_grad(self.mock_monitor, self.torch_opt.param_names) + for param, name in self.torch_opt.param_names.items(): + group_index, param_id = [int(i) for i in name.replace('param','').split('_')] + self.assertTrue(torch.equal(result[name], self.torch_opt.averaged_gradients[group_index][param_id])) - self.params2name = {'param1': 'weight', 'param2': 'bias'} - self.name2index = {'weight': 0, 'bias': 1} + def test_fetch_mv(self): + result = self.optimizer_mon.fetch_mv(self.mock_monitor, self.torch_opt.param_names) + for param, name in self.torch_opt.param_names.items(): + self.assertTrue(torch.equal(result.exp_avg[name], torch.ones_like(param).flatten())) + self.assertTrue(torch.equal(result.exp_avg_sq[name], torch.ones_like(param).flatten())) - self.optimizer_monitor = DeepSpeedZeroOptimizerStage1or2Mon() - self.torch_opt = MagicMock() - self.torch_opt.groups_padding = [1, 2, 3] - self.torch_opt.single_partition_of_fp32_groups = [torch.tensor([1, 2]), torch.tensor([3, 4, 5])] - self.torch_opt.bit16_groups = [ - [torch.tensor([6, 7]), torch.tensor([8])], - [torch.tensor([9, 10, 11])] - ] - - name2indices = self.optimizer_monitor.get_param_index(self.params2name, self.name2index, self.torch_opt) - for name, indices in name2indices.items(): - self.assertIn(name, self.params2name.values()) - self.assertIsInstance(indices, tuple) - self.assertEqual(len(indices), 4) +class TestDeepSpeedZeroOptimizerStage3Mon(unittest.TestCase): + def setUp(self): + bit16_groups, param_names, _, grad_position = setup_param_groups() + + mock_opt = MagicMock() + mock_opt.param_names = param_names + mock_opt.fp16_groups = bit16_groups + mock_opt.fp32_partitioned_groups_flat = [torch.stack(group,dim=0).flatten().float() + for group in bit16_groups] + mock_opt.averaged_gradients = {group_idx: [torch.randn_like(param) for param in group] + for group_idx, group in enumerate(bit16_groups)} + mock_opt.grad_position = grad_position + mock_opt.get_param_id = lambda x: int(param_names[x].split('_')[1]) + mock_opt.state = { + flat_group: { + 'exp_avg': torch.ones_like(flat_group), + 'exp_avg_sq': torch.ones_like(flat_group) + } for flat_group in mock_opt.fp32_partitioned_groups_flat + } + + self.torch_opt = mock_opt + self.optimizer_mon = DeepSpeedZeroOptimizerStage3Mon(mock_opt) + self.mock_monitor = setup_mock_monitor() + + def test_fetch_grad(self): + self.mock_monitor.name2tag = {name:{MonitorConst.POST_GRAD: name} for name in self.torch_opt.param_names.values()} + result = self.optimizer_mon.fetch_grad(self.mock_monitor, self.torch_opt.param_names) + for param, name in self.torch_opt.param_names.items(): + group_index, param_id = [int(i) for i in name.replace('param','').split('_')] + self.assertTrue(torch.equal(result[name], self.torch_opt.averaged_gradients[group_index][param_id])) def test_fetch_mv(self): - self.monitor = MagicMock() - self.torch_opt = MagicMock() - self.params2name = MagicMock() - self.torch_opt.fp16_partitioned_groups = MagicMock() - self.optimizer = DeepSpeedZeroOptimizerStage1or2Mon() - - # mock _fetch_mv_grad_in_adam - mv_result = MVGradResult(exp_avg={}, exp_avg_sq={}, update={}, ratio={}, grad={}) - self.mock_fetch_mv_grad_in_adam = MagicMock(return_value=mv_result) - self.optimizer._fetch_mv_grad_in_adam = self.mock_fetch_mv_grad_in_adam - - res = self.optimizer.fetch_mv(self.monitor, self.torch_opt, self.params2name) - self.assertIsInstance(res, MVGradResult) + result = self.optimizer_mon.fetch_mv(self.mock_monitor, self.torch_opt.param_names) + for param, name in self.torch_opt.param_names.items(): + self.assertTrue(torch.equal(result.exp_avg[name], torch.ones_like(param).flatten())) + self.assertTrue(torch.equal(result.exp_avg_sq[name], torch.ones_like(param).flatten())) class TestOptimizerMonFactory(unittest.TestCase): @@ -291,48 +365,48 @@ class TestOptimizerMonFactory(unittest.TestCase): mix_optimizer_class = MagicMock() mix_optimizer_class.__name__ = "Float16OptimizerWithFloat16Params" mix_optimizer.__class__ = mix_optimizer_class - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(mix_optimizer)[0], + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(mix_optimizer), MixPrecisionOptimizerMon) dis_optimizer = MagicMock() dis_optimizer_class = MagicMock() dis_optimizer_class.__name__ = "DistributedOptimizer" dis_optimizer.__class__ = dis_optimizer_class - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(dis_optimizer)[0], + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(dis_optimizer), MegatronDistributedOptimizerMon) fp32_optimizer = MagicMock() fp32_optimizer_class = MagicMock() fp32_optimizer_class.__name__ = "FP32Optimizer" fp32_optimizer.__class__ = fp32_optimizer_class - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(fp32_optimizer)[0], - MegatronFP32OptimizerMon) + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(fp32_optimizer), + OptimizerMon) chained_optimizer = MagicMock() chained_optimizer_class = MagicMock() chained_optimizer_class.__name__ = "ChainedOptimizer" chained_optimizer.__class__ = chained_optimizer_class chained_optimizer.chained_optimizers = [mix_optimizer, mix_optimizer] - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(chained_optimizer)[0], + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(chained_optimizer), MegatronChainedMixPrecisionOptimizerMon) chained_optimizer.chained_optimizers = [dis_optimizer, dis_optimizer] - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(chained_optimizer)[0], + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(chained_optimizer), MegatronChainedDistributedOptimizerMon) deepspeed_optimizer = MagicMock() deepspeed_optimizer_class = MagicMock() deepspeed_optimizer_class.__name__ = "BF16_Optimizer" deepspeed_optimizer.__class__ = deepspeed_optimizer_class - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(deepspeed_optimizer)[0], + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(deepspeed_optimizer), DeepSpeedZeroOptimizerStage0Mon) deepspeed_optimizer_class.__name__ = "DeepSpeedZeroOptimizer" - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(deepspeed_optimizer)[0], + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(deepspeed_optimizer), DeepSpeedZeroOptimizerStage1or2Mon) deepspeed_optimizer_class.__name__ = "DeepSpeedZeroOptimizer_Stage3" - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(deepspeed_optimizer)[0], + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(deepspeed_optimizer), DeepSpeedZeroOptimizerStage3Mon) - # 测试未知的优化器类型，应该返回DummyOptimizerMon + # 测试未知的优化器类型，应该返回OptimizerMon unknown_optimizer = MagicMock() unknown_optimizer_class = MagicMock() unknown_optimizer_class.__name__ = "unknown" unknown_optimizer.__class__ = unknown_optimizer_class - self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(unknown_optimizer)[0], DummyOptimizerMon) + self.assertIsInstance(OptimizerMonFactory.create_optimizer_mon(unknown_optimizer), OptimizerMon) if __name__ == '__main__': diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/parse_tool/test_interactive_cli.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/parse_tool/test_interactive_cli.py index b875bd7e8e17f6b869b2a1b1498982b2a17e1258..3a09d41588a94043f54161023ffbba573c60d76c 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/parse_tool/test_interactive_cli.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/parse_tool/test_interactive_cli.py @@ -26,63 +26,50 @@ class TestInteractiveCli(unittest.TestCase): @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.ParseTool.prepare', return_value=None) def test_prepare(self, mock_prepare): self.interactive_cli.prepare() - mock_prepare.assert_called_once() - @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.execute_command', return_value=None) - def test_default(self, mock_execute_command): - res = self.interactive_cli.default() - - mock_execute_command.assert_called_once() - self.assertFalse(res) - - @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.execute_command', return_value=None) - def test_do_run(self, mock_execute_command): - self.interactive_cli.do_run() - - mock_execute_command.assert_called_once() + def test_default(self, command='rm'): + res = self.interactive_cli.default(command) + self.assertIsNone(res) @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.ParseTool.do_compare_converted_dir') @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.ParseTool.do_vector_compare') def test_do_vc(self, mock_do_vector_compare, mock_do_compare_converted_dir): - with patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.check_path_valid'), \ - patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.check_files_in_path'): - with patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.dir_contains_only', return_value=False): - self.interactive_cli.do_vc('-m my_dump_path -g golden_dump_path -out output_path -cmp_path msaccucmp_path') - + with (patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.check_path_valid'), + patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.check_files_in_path')): + with patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.dir_contains_only', + return_value=False): + self.interactive_cli.do_vc( + '-m my_dump_path -g golden_dump_path -out output_path -cmp_path msaccucmp_path') mock_do_vector_compare.assert_called_once() - with patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.dir_contains_only', return_value=True): - self.interactive_cli.do_vc('-m my_dump_path -g golden_dump_path -out output_path -cmp_path msaccucmp_path') - + with patch('msprobe.pytorch.parse_tool.lib.interactive_cli.Util.dir_contains_only', + return_value=True): + self.interactive_cli.do_vc( + '-m my_dump_path -g golden_dump_path -out output_path -cmp_path msaccucmp_path') mock_do_compare_converted_dir.assert_called_once() @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.ParseTool.do_convert_dump', return_value=None) def test_do_dc(self, mock_do_convert_dump): self.interactive_cli.do_dc('-n file_name/file_path -f format -out output_path') - mock_do_convert_dump.assert_called_once() @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.ParseTool.do_print_data', return_value=None) def test_do_pt(self, mock_do_print_data): self.interactive_cli.do_pt('-n file_path') - mock_do_print_data.assert_called_once() @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.ParseTool.do_parse_pkl', return_value=None) def test_do_pk(self, mock_do_parse_pkl): self.interactive_cli.do_pk('-f pkl_path -n api_name') - mock_do_parse_pkl.assert_called_once() @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.ParseTool.do_compare_data', return_value=None) def test_do_cn(self, mock_do_comapre_data): self.interactive_cli.do_cn('-m my_data*.npy -g golden*.npu -p num -al atol -rl rtol') - mock_do_comapre_data.assert_called_once() @patch('msprobe.pytorch.parse_tool.lib.interactive_cli.ParseTool.do_convert_api_dir', return_value=None) def test_do_cad(self, mock_do_convert_api_dir): self.interactive_cli.do_cad('-m my_dump_path -out output_path -asc msaccucmp_path') - mock_do_convert_api_dir.assert_called_once() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/parse_tool/test_parse_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/parse_tool/test_parse_utils.py index dfec4d20366c6e834939130009dc6d33d1cbe9ed..c148f84d0d20213631e9be039521a14d970849e9 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/parse_tool/test_parse_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/parse_tool/test_parse_utils.py @@ -88,7 +88,7 @@ class TestUtils(unittest.TestCase): obj = np.array([1, 2, 3, 4, 5]) res = self.util.get_md5_for_numpy(obj) - self.assertEqual(res, '3cd8e13ca72251bfd8c08e209abcf46f') + self.assertEqual(res, 'baa24928') def test_deal_with_dir_or_file_inconsistency(self): with self.assertRaises(ParseException): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py index 0724581bc79f48ed158b691d315a5a75cc1bc65d..f12cffd8da88ab3a42c471eb5a1c6197ef59d634 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py @@ -181,7 +181,7 @@ class TestStatisticsConfig(unittest.TestCase): self.config.summary_mode = "invalid_mode" with self.assertRaises(Exception) as context: self.config._check_summary_mode() - self.assertIn(str(context.exception), "summary_mode is invalid") + self.assertIn(str(context.exception), "[msprobe] 无效参数：") def test_check_summary_mode_none(self): self.config.summary_mode = None @@ -268,7 +268,7 @@ class TestFreeBenchmarkCheckConfig(unittest.TestCase): invalid_config["fuzz_device"] = "cpu" invalid_config["pert_mode"] = "INVALID_CPU_MODE" config = FreeBenchmarkCheckConfig(invalid_config) - self.assertIn("You neet to and can only set fuzz_device as ", str(mock_error.call_args)) + self.assertIn("You need to and can only set fuzz_device as ", str(mock_error.call_args)) @patch('msprobe.core.common.log.logger.error_log_with_exp') def test_check_handler_type_invalid(self, mock_error): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_debug_save.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_debug_save.py index 534437260e66d9e586d69d557d30e308a9f4f3ee..cf7aec0ed1bc4147cd5ee1a56ecbc686bba33a54 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_debug_save.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_debug_save.py @@ -36,13 +36,15 @@ class TestPytorchDebuggerSave(TestCase): } common_config = CommonConfig(statistics_task_json) task_config = BaseConfig(statistics_task_json) - with patch("msprobe.pytorch.debugger.precision_debugger.parse_json_config", return_value=(common_config, task_config)): + with patch("msprobe.pytorch.debugger.precision_debugger.parse_json_config", + return_value=(common_config, task_config)): self.debugger = PrecisionDebugger() def test_forward_and_backward(self): def forward_func(x, y): PrecisionDebugger.save(x, "x_tensor") return x * y + x = torch.tensor([1.]) y = torch.tensor([2.]) x.requires_grad = True @@ -53,28 +55,28 @@ class TestPytorchDebuggerSave(TestCase): "framework": "pytorch", "dump_data_dir": None, "data": { - "x_tensor.0": { + "x_tensor.0.debug": { "type": "torch.Tensor", "dtype": "torch.float32", "shape": torch.Size([1]), - "Max": 1.0, - "Min": 1.0, - "Mean": 1.0, - "Norm": 1.0, "requires_grad": True }, - "x_tensor_grad.0": { + "x_tensor_grad.0.debug": { "type": "torch.Tensor", "dtype": "torch.float32", "shape": torch.Size([1]), - "Max": 2.0, - "Min": 2.0, - "Mean": 2.0, - "Norm": 2.0, "requires_grad": False } } } + loss = forward_func(x, y) loss.backward() - self.assertEqual(self.debugger.service.data_collector.data_writer.cache_debug, result_json) \ No newline at end of file + + result = self.debugger.service.data_collector.data_writer.cache_debug + # Remove 'tensor_stat_index' from all entries in the data dictionary + for key in result["data"]: + if 'tensor_stat_index' in result["data"][key]: + del result["data"][key]['tensor_stat_index'] + + self.assertEqual(result, result_json) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/visualization_ut/compare/test_graph_comparator.py b/debug/accuracy_tools/msprobe/test/visualization_ut/compare/test_graph_comparator.py index f4d68ccb530919dbdfedaa12bea716b2c70e278d..4accdacd76a434b6329a9fce378e38927092e9ae 100644 --- a/debug/accuracy_tools/msprobe/test/visualization_ut/compare/test_graph_comparator.py +++ b/debug/accuracy_tools/msprobe/test/visualization_ut/compare/test_graph_comparator.py @@ -1,5 +1,6 @@ import os import unittest +from typing import Any from dataclasses import dataclass from unittest.mock import patch from unittest.mock import MagicMock @@ -12,7 +13,7 @@ from msprobe.visualization.utils import GraphConst class Args: input_path: str = None output_path: str = None - layer_mapping: str = None + layer_mapping: Any = None framework: str = None overflow_check: bool = False fuzzy_match: bool = False @@ -39,7 +40,7 @@ class TestGraphComparator(unittest.TestCase): mock_load_data_json_file.return_value = "data_dict" mock_load_json_file.return_value = "construct_dict" mock_get_compare_mode.return_value = GraphConst.SUMMARY_COMPARE - self.comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path)) + self.comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path), False) self.comparator._parse_param(self.dump_path_param, self.output_path) self.assertEqual(self.comparator.dump_path_param, { @@ -57,7 +58,7 @@ class TestGraphComparator(unittest.TestCase): mock_load_data_json_file.return_value = "data_dict" mock_load_json_file.return_value = "construct_dict" mock_get_compare_mode.return_value = GraphConst.SUMMARY_COMPARE - comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path)) + comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path), False) comparator._compare_nodes = MagicMock() comparator._postcompare = MagicMock() @@ -76,7 +77,7 @@ class TestGraphComparator(unittest.TestCase): node = MagicMock() compare_result_list = [("output1", "data1"), ("input1", "data2")] - comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path)) + comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path), False) comparator.ma = MagicMock() comparator.ma.prepare_real_data.return_value = True @@ -100,7 +101,7 @@ class TestGraphComparator(unittest.TestCase): mock_run_real_data.return_value = mock_df mock_get_csv_df.return_value = mock_df mock_get_node_error_status.return_value = True - comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path)) + comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path), False) comparator.ma = MagicMock() comparator.ma.compare_mode = GraphConst.REAL_DATA_COMPARE comparator._handle_api_collection_index = MagicMock() @@ -118,7 +119,7 @@ class TestGraphComparator(unittest.TestCase): mock_load_data_json_file.return_value = "data_dict" mock_load_json_file.return_value = "construct_dict" mock_get_compare_mode.return_value = GraphConst.SUMMARY_COMPARE - comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path)) + comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path), False) apis = BaseNode(NodeOp.api_collection, 'Apis_Between_Modules.0') api1 = BaseNode(NodeOp.function_api, 'Tensor.a.0') api1.data = {GraphConst.JSON_INDEX_KEY: 0.9} @@ -145,11 +146,12 @@ class TestGraphComparator(unittest.TestCase): mock_get_compare_mode.return_value = GraphConst.SUMMARY_COMPARE mock_mapping_match.return_value = (node_b, [], []) mock_compare_node.return_value = ['result'] - comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path)) + comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path, layer_mapping=True), True) comparator.mapping_dict = True comparator._compare_nodes(node_n) self.assertEqual(node_n.matched_node_link, ['Tensor.b.0']) self.assertEqual(node_b.matched_node_link, ['Tensor.a.0']) + comparator = GraphComparator(self.graphs, self.dump_path_param, Args(output_path=self.output_path), False) comparator.mapping_dict = False node_n = BaseNode(NodeOp.function_api, 'Tensor.a.0') node_b = BaseNode(NodeOp.function_api, 'Tensor.a.0') @@ -185,6 +187,6 @@ class TestGraphComparator(unittest.TestCase): 'stack_json_path': os.path.join(dir_name, 'input', 'step0', 'rank0', 'stack.json'), 'is_print_compare_log': True } - comparator = GraphComparator(self.graphs, dump_path_param, Args(output_path=self.output_path)) + comparator = GraphComparator(self.graphs, dump_path_param, Args(output_path=self.output_path), False) comparator.add_compare_result_to_node(node, compare_result_list) self.assertEqual(node.data, {'precision_index': 0}) diff --git a/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step0/rank0/dump.json b/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step0/rank0/dump.json index 330122252bd65cb01bbf9f0cd6c912f407b32a28..946af014ad298b391509c9608a321c16ab8c1453 100644 --- a/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step0/rank0/dump.json +++ b/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step0/rank0/dump.json @@ -2,5 +2,6 @@ "task": "statistics", "level": "mix", "dump_data_dir": null, - "data": {} + "data": {}, + "framework": "pytorch" } diff --git a/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step0/rank1/dump.json b/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step0/rank1/dump.json index 330122252bd65cb01bbf9f0cd6c912f407b32a28..946af014ad298b391509c9608a321c16ab8c1453 100644 --- a/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step0/rank1/dump.json +++ b/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step0/rank1/dump.json @@ -2,5 +2,6 @@ "task": "statistics", "level": "mix", "dump_data_dir": null, - "data": {} + "data": {}, + "framework": "pytorch" } diff --git a/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step1/rank0/dump.json b/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step1/rank0/dump.json index 330122252bd65cb01bbf9f0cd6c912f407b32a28..946af014ad298b391509c9608a321c16ab8c1453 100644 --- a/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step1/rank0/dump.json +++ b/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step1/rank0/dump.json @@ -2,5 +2,6 @@ "task": "statistics", "level": "mix", "dump_data_dir": null, - "data": {} + "data": {}, + "framework": "pytorch" } diff --git a/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step2/rank0/dump.json b/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step2/rank0/dump.json index 330122252bd65cb01bbf9f0cd6c912f407b32a28..946af014ad298b391509c9608a321c16ab8c1453 100644 --- a/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step2/rank0/dump.json +++ b/debug/accuracy_tools/msprobe/test/visualization_ut/input_format_correct/step2/rank0/dump.json @@ -2,5 +2,6 @@ "task": "statistics", "level": "mix", "dump_data_dir": null, - "data": {} + "data": {}, + "framework": "pytorch" } diff --git a/debug/accuracy_tools/msprobe/test/visualization_ut/test_graph_service.py b/debug/accuracy_tools/msprobe/test/visualization_ut/test_graph_service.py index 7dfd9564ebc21327f3e7e29be90da7f78c3b0393..f9ca5592aaa153bc0446443548c3e18329784a18 100644 --- a/debug/accuracy_tools/msprobe/test/visualization_ut/test_graph_service.py +++ b/debug/accuracy_tools/msprobe/test/visualization_ut/test_graph_service.py @@ -7,7 +7,7 @@ import argparse from dataclasses import dataclass from unittest.mock import patch -from msprobe.visualization.graph_service import _compare_graph, _build_graph, _compare_graph_ranks, \ +from msprobe.visualization.graph_service import _compare_graph_result, _build_graph_result, _compare_graph_ranks, \ _compare_graph_steps, _build_graph_ranks, _build_graph_steps, _graph_service_command, _graph_service_parser from msprobe.core.common.utils import CompareException @@ -45,30 +45,31 @@ class TestGraphService(unittest.TestCase): last_call_args = mock_log_info.call_args[0][0] self.assertIn(log_info, last_call_args) matches = re.findall(self.pattern, last_call_args) - self.assertTrue(os.path.exists(os.path.join(self.output, matches[0]))) + if matches: + self.assertTrue(os.path.exists(os.path.join(self.output, matches[0]))) @patch('msprobe.core.common.log.logger.info') - def test_compare_graph(self, mock_log_info): + def test_compare_graph_result(self, mock_log_info): args = Args(output_path=self.output, framework='pytorch') - result = _compare_graph(self.input_param, args) + result = _compare_graph_result(self.input_param, args) self.assertEqual(mock_log_info.call_count, 2) self.assertIsNotNone(result) args = Args(output_path=self.output, framework='mindspore') - result = _compare_graph(self.input_param, args) + result = _compare_graph_result(self.input_param, args) self.assertIsNotNone(result) args = Args(output_path=self.output, framework='pytorch', layer_mapping=self.layer_mapping) - result = _compare_graph(self.input_param, args) + result = _compare_graph_result(self.input_param, args) self.assertIsNotNone(result) args = Args(output_path=self.output, framework='pytorch', overflow_check=True) - result = _compare_graph(self.input_param, args) + result = _compare_graph_result(self.input_param, args) self.assertIsNotNone(result) @patch('msprobe.core.common.log.logger.info') - def test_build_graph(self, mock_log_info): - result = _build_graph(os.path.join(self.input, 'step0', 'rank0'), Args(overflow_check=True)) + def test_build_graph_result(self, mock_log_info): + result = _build_graph_result(os.path.join(self.input, 'step0', 'rank0'), Args(overflow_check=True)) self.assertEqual(mock_log_info.call_count, 1) self.assertIsNotNone(result) @@ -81,7 +82,7 @@ class TestGraphService(unittest.TestCase): } args = Args(output_path=self.output, framework='pytorch') _compare_graph_ranks(input_param, args) - self.assert_log_info(mock_log_info) + self.assert_log_info(mock_log_info, 'Successfully exported compare graph results.') input_param1 = { 'npu_path': os.path.join(self.input, 'step0'), @@ -101,7 +102,7 @@ class TestGraphService(unittest.TestCase): } args = Args(output_path=self.output, framework='pytorch') _compare_graph_steps(input_param, args) - self.assert_log_info(mock_log_info) + self.assert_log_info(mock_log_info, 'Successfully exported compare graph results.') input_param1 = { 'npu_path': self.input, @@ -115,12 +116,12 @@ class TestGraphService(unittest.TestCase): @patch('msprobe.core.common.log.logger.info') def test_build_graph_ranks(self, mock_log_info): _build_graph_ranks(os.path.join(self.input, 'step0'), Args(output_path=self.output)) - self.assert_log_info(mock_log_info, "Model graph built successfully, the result file is saved in") + self.assert_log_info(mock_log_info, "Successfully exported build graph results.") @patch('msprobe.core.common.log.logger.info') def test_build_graph_steps(self, mock_log_info): _build_graph_steps(self.input, Args(output_path=self.output)) - self.assert_log_info(mock_log_info, "Model graph built successfully, the result file is saved in") + self.assert_log_info(mock_log_info, "Successfully exported build graph results.") @patch('msprobe.core.common.log.logger.info') def test_graph_service_command(self, mock_log_info): @@ -129,7 +130,7 @@ class TestGraphService(unittest.TestCase): args = Args(input_path=self.output_json[0], output_path=self.output, framework='pytorch') _graph_service_command(args) - self.assert_log_info(mock_log_info) + self.assert_log_info(mock_log_info, 'Exporting compare graph result successfully, the result file is saved in') input_param1 = { 'npu_path': os.path.join(self.input, 'step0', 'rank0'), @@ -139,7 +140,7 @@ class TestGraphService(unittest.TestCase): json.dump(input_param1, f, indent=4) args = Args(input_path=self.output_json[1], output_path=self.output, framework='pytorch') _graph_service_command(args) - self.assert_log_info(mock_log_info, "Model graph built successfully, the result file is saved in") + self.assert_log_info(mock_log_info, "Model graph exported successfully, the result file is saved in") input_param2 = { 'npu_path': os.path.join(self.input, 'step0'), @@ -150,7 +151,7 @@ class TestGraphService(unittest.TestCase): json.dump(input_param2, f, indent=4) args = Args(input_path=self.output_json[2], output_path=self.output, framework='pytorch') _graph_service_command(args) - self.assert_log_info(mock_log_info) + self.assert_log_info(mock_log_info, 'Successfully exported compare graph results.') input_param3 = { 'npu_path': self.input, @@ -161,7 +162,7 @@ class TestGraphService(unittest.TestCase): json.dump(input_param3, f, indent=4) args = Args(input_path=self.output_json[3], output_path=self.output, framework='pytorch') _graph_service_command(args) - self.assert_log_info(mock_log_info) + self.assert_log_info(mock_log_info, 'Successfully exported compare graph results.') input_param4 = { 'npu_path': os.path.join(self.input, 'step0'), @@ -171,7 +172,7 @@ class TestGraphService(unittest.TestCase): json.dump(input_param4, f, indent=4) args = Args(input_path=self.output_json[4], output_path=self.output, framework='pytorch') _graph_service_command(args) - self.assert_log_info(mock_log_info, "Model graph built successfully, the result file is saved in") + self.assert_log_info(mock_log_info, "Successfully exported build graph results.") input_param5 = { 'npu_path': self.input, @@ -181,7 +182,7 @@ class TestGraphService(unittest.TestCase): json.dump(input_param5, f, indent=4) args = Args(input_path=self.output_json[5], output_path=self.output, framework='pytorch') _graph_service_command(args) - self.assert_log_info(mock_log_info, "Model graph built successfully, the result file is saved in") + self.assert_log_info(mock_log_info, "Successfully exported build graph results.") input_param6 = { 'npu_path': self.input, diff --git a/debug/accuracy_tools/msprobe/test/visualization_ut/test_visualization_utils.py b/debug/accuracy_tools/msprobe/test/visualization_ut/test_visualization_utils.py index e5b0afaadf9def910c248b945ad15084300a65c0..41ea145208dc658a83bb5c791d6b05a0abb30616 100644 --- a/debug/accuracy_tools/msprobe/test/visualization_ut/test_visualization_utils.py +++ b/debug/accuracy_tools/msprobe/test/visualization_ut/test_visualization_utils.py @@ -1,7 +1,7 @@ import os import unittest from msprobe.visualization.utils import (load_json_file, load_data_json_file, str2float, check_directory_content, - GraphConst) + GraphConst, SerializableArgs) class TestMappingConfig(unittest.TestCase): @@ -37,6 +37,21 @@ class TestMappingConfig(unittest.TestCase): input_type = check_directory_content(os.path.join(self.input, "step0", "rank0")) self.assertEqual(input_type, GraphConst.FILES) + def test_serializable_args(self): + class TmpArgs: + def __init__(self, a, b, c): + self.a = a + self.b = b + self.c = c + input_args1 = TmpArgs('a', 123, [1, 2, 3]) + serializable_args1 = SerializableArgs(input_args1) + self.assertEqual(serializable_args1.__dict__, input_args1.__dict__) + input_args2 = TmpArgs('a', 123, lambda x: print(x)) + serializable_args2 = SerializableArgs(input_args2) + self.assertNotEqual(serializable_args2.__dict__, input_args2.__dict__) + + + if __name__ == '__main__': unittest.main() diff --git a/debug/accuracy_tools/msprobe/visualization/builder/graph_builder.py b/debug/accuracy_tools/msprobe/visualization/builder/graph_builder.py index bec99d675f4b1238fde3905037ec5f7fb5a0c8fe..78b4b83cb17c99a80dfbc6eeb9ceafba1543fedf 100644 --- a/debug/accuracy_tools/msprobe/visualization/builder/graph_builder.py +++ b/debug/accuracy_tools/msprobe/visualization/builder/graph_builder.py @@ -14,9 +14,11 @@ # limitations under the License. import re +from dataclasses import dataclass from msprobe.core.common.const import Const from msprobe.core.common.file_utils import load_json, save_json +from msprobe.core.common.utils import load_stack_json from msprobe.visualization.builder.msprobe_adapter import get_input_output from msprobe.visualization.builder.msprobe_adapter import op_patterns from msprobe.visualization.graph.graph import Graph @@ -44,7 +46,7 @@ class GraphBuilder: """ construct_dict = load_json(construct_path) dump_dict = load_json(data_path) - stack_dict = load_json(stack_path) + stack_dict = load_stack_json(stack_path) if not complete_stack: GraphBuilder._simplify_stack(stack_dict) data_dict = dump_dict.get(GraphConst.DATA_KEY, {}) @@ -61,10 +63,10 @@ class GraphBuilder: """ result = {} if config.graph_b: - result[GraphConst.JSON_NPU_KEY] = config.graph_n.to_dict() - result[GraphConst.JSON_BENCH_KEY] = config.graph_b.to_dict() + result[GraphConst.JSON_NPU_KEY] = config.graph_n.to_dict(config.compare_mode) + result[GraphConst.JSON_BENCH_KEY] = config.graph_b.to_dict(config.compare_mode) else: - result = config.graph_n.to_dict() + result = config.graph_n.to_dict(config.compare_mode) if config.tool_tip: result[GraphConst.JSON_TIP_KEY] = config.tool_tip if config.node_colors: @@ -277,7 +279,7 @@ class GraphBuilder: class GraphExportConfig: def __init__(self, graph_n, graph_b=None, tool_tip=None, node_colors=None, micro_steps=None, task='', - overflow_check=False): + overflow_check=False, compare_mode=None): self.graph_n = graph_n self.graph_b = graph_b self.tool_tip = tool_tip @@ -285,3 +287,21 @@ class GraphExportConfig: self.micro_steps = micro_steps self.task = task self.overflow_check = overflow_check + self.compare_mode = compare_mode + + +@dataclass +class GraphInfo: + graph: Graph + construct_path: str + data_path: str + stack_path: str + + +@dataclass +class BuildGraphTaskInfo: + graph_info_n: GraphInfo + graph_info_b: GraphInfo + npu_rank: str + bench_rank: str + time_str: str diff --git a/debug/accuracy_tools/msprobe/visualization/builder/msprobe_adapter.py b/debug/accuracy_tools/msprobe/visualization/builder/msprobe_adapter.py index 751006f3e527726ce049e054ab9cbb2ad87de064..2b7f7886535068824e782c8cfab1b6aa283198e5 100644 --- a/debug/accuracy_tools/msprobe/visualization/builder/msprobe_adapter.py +++ b/debug/accuracy_tools/msprobe/visualization/builder/msprobe_adapter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,12 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import re -from msprobe.core.compare.acc_compare import read_op, merge_tensor, get_accuracy + +from msprobe.core.compare.acc_compare import ModeConfig +from msprobe.core.compare.multiprocessing_compute import CompareRealData +from msprobe.core.compare.utils import read_op, merge_tensor, get_accuracy, make_result_table from msprobe.core.common.utils import set_dump_path, get_dump_mode from msprobe.visualization.utils import GraphConst from msprobe.core.common.const import Const -from msprobe.core.compare.acc_compare import ModeConfig + # 用于将节点名字解析成对应的NodeOp的规则 op_patterns = [ @@ -53,13 +57,11 @@ def run_real_data(dump_path_param, csv_path, framework, is_cross_frame=False): mode_config = ModeConfig(stack_mode=False, auto_analyze=True, fuzzy_match=False, dump_mode=Const.ALL) if framework == Const.PT_FRAMEWORK: - from msprobe.pytorch.compare.pt_compare import PTComparator - return PTComparator(mode_config).do_multi_process(dump_path_param, csv_path) + from msprobe.pytorch.compare.pt_compare import read_real_data + return CompareRealData(read_real_data, mode_config, is_cross_frame).do_multi_process(dump_path_param, csv_path) else: - from msprobe.mindspore.compare.ms_compare import MSComparator, MappingConfig - ms_comparator = MSComparator(mode_config, MappingConfig()) - ms_comparator.cross_frame = is_cross_frame - return ms_comparator.do_multi_process(dump_path_param, csv_path) + from msprobe.mindspore.compare.ms_compare import read_real_data + return CompareRealData(read_real_data, mode_config, is_cross_frame).do_multi_process(dump_path_param, csv_path) def get_input_output(node_data, node_id): @@ -119,11 +121,13 @@ def compare_data_fuzzy(data_dict_list1, data_dict_list2): return True -def format_node_data(data_dict, node_id=None): +def format_node_data(data_dict, node_id=None, compare_mode=None): """ 删除节点数据中不需要展示的字段 """ del_list = ['requires_grad', 'full_op_name'] + if GraphConst.MD5_COMPARE != compare_mode: + del_list.append(Const.MD5) if node_id and GraphConst.BATCH_P2P in node_id: del_list.extend(['op', 'peer', 'tag', 'group_id']) for _, value in data_dict.items(): @@ -171,7 +175,7 @@ def _format_decimal_string(s): """ 使用正则表达式匹配包含数字、小数点和可选的百分号的字符串 """ - pattern = re.compile(r'\d{1,20}\.\d{1,20}%?') + pattern = re.compile(r'^\d{1,20}\.\d{1,20}%?$') matches = pattern.findall(s) for match in matches: is_percent = match.endswith('%') @@ -226,3 +230,12 @@ def _format_data(data_dict): if all_null: data_dict.clear() data_dict[GraphConst.VALUE] = GraphConst.NULL + + +def get_csv_df(stack_mode, csv_data, compare_mode): + """ + 调用acc接口写入csv + """ + + dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode) + return make_result_table(csv_data, dump_mode, stack_mode) diff --git a/debug/accuracy_tools/msprobe/visualization/compare/graph_comparator.py b/debug/accuracy_tools/msprobe/visualization/compare/graph_comparator.py index 3f695d23483c8980c958995a36025b0514877cf9..95982658d2f431463476912e9c229b281f817861 100644 --- a/debug/accuracy_tools/msprobe/visualization/compare/graph_comparator.py +++ b/debug/accuracy_tools/msprobe/visualization/compare/graph_comparator.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,25 +14,27 @@ # limitations under the License. import re -from msprobe.visualization.builder.msprobe_adapter import compare_node, get_compare_mode, run_real_data -from msprobe.visualization.utils import GraphConst, load_json_file, load_data_json_file, get_csv_df +from msprobe.visualization.builder.msprobe_adapter import compare_node, get_compare_mode, run_real_data, get_csv_df +from msprobe.visualization.utils import GraphConst, load_json_file, load_data_json_file from msprobe.visualization.graph.graph import Graph, NodeOp from msprobe.visualization.compare.mode_adapter import ModeAdapter from msprobe.core.common.const import Const -from msprobe.core.common.utils import recursion_depth_decorator +from msprobe.core.common.decorator import recursion_depth_decorator class GraphComparator: MAX_DEPTH = 1000 - def __init__(self, graphs, dump_path_param, args, mapping_dict=None): + def __init__(self, graphs, dump_path_param, args, is_cross_framework, mapping_dict=None): self.graph_n = graphs[0] self.graph_b = graphs[1] self._parse_param(dump_path_param, args.output_path) self.framework = args.framework + self.layer_mapping = args.layer_mapping self.mapping_dict = mapping_dict self.fuzzy_match = args.fuzzy_match self.pattern = re.compile(r'\.\d+\.') + self.is_cross_framework = is_cross_framework def compare(self): """ @@ -69,50 +71,56 @@ class GraphComparator: node.data[GraphConst.JSON_INDEX_KEY] = precision_index node.data.update(other_dict) - @recursion_depth_decorator('GraphComparator._compare_nodes', max_depth=MAX_DEPTH) - def _compare_nodes(self, node_n): + def _compare_nodes(self, node_root): """ - 递归遍历NPU树中的节点，如果在Bench中找到具有相同名称的节点，检查他们的祖先和参数信息，检查一致则及逆行精度数据对比 + 遍历NPU树中的节点，如果在Bench中找到具有相同名称的节点，检查他们的祖先和参数信息，检查一致则及逆行精度数据对比 这里采用先序遍历，好处在于当这个节点被比较时，他的先序已经被匹配，这可以为后续的模糊匹配提供重要信息 """ - if self.mapping_dict: - node_b, ancestors_n, ancestors_b = Graph.mapping_match(node_n, self.graph_b, self.mapping_dict) - if node_b: - ancestors_n.append(node_n.id) - ancestors_b.append(node_b.id) - node_n.matched_node_link = ancestors_b - node_b.matched_node_link = ancestors_n - else: - node_b, ancestors = Graph.match(self.graph_n, node_n, self.graph_b) - if node_b: - ancestors.append(node_b.id) - node_n.add_link(node_b, ancestors) - if node_b: - # 真实数据比对只会得到基本信息，并没有精度指标，需要调用多进程对比接口 - self._get_and_add_result(node_n, node_b) - for subnode in node_n.subnodes: - self._compare_nodes(subnode) - - @recursion_depth_decorator('GraphComparator._compare_nodes_fuzzy', max_depth=MAX_DEPTH) - def _compare_nodes_fuzzy(self, node_n): - if node_n.op != NodeOp.function_api: - # 模块经过模糊匹配 - node_b, ancestors_n, ancestors_b = Graph.fuzzy_match(node_n, self.graph_b.node_map.get(node_n.id)) + def compare_single_node(node_n): + if self.layer_mapping: + node_b, ancestors_n, ancestors_b = Graph.mapping_match(node_n, self.graph_b, self.mapping_dict) + if node_b: + ancestors_n.append(node_n.id) + ancestors_b.append(node_b.id) + node_n.matched_node_link = ancestors_b + node_b.matched_node_link = ancestors_n + else: + node_b, ancestors = Graph.match(self.graph_n, node_n, self.graph_b) + if node_b: + ancestors.append(node_b.id) + node_n.add_link(node_b, ancestors) if node_b: - self._process_matched_nodes(node_n, node_b, ancestors_n, ancestors_b) - # 匹配上的两个模块中的所有api, 忽略dump调用次数，按照名称一致+模块中的调用顺序进行匹配 - recount_result_n = self._recount_api_node(node_n) - recount_result_b = self._recount_api_node(node_b) - for recount_node_id, node_id_n in recount_result_n.items(): - api_node_n = self.graph_n.node_map.get(node_id_n) - if not api_node_n: - continue - api_node_b, ancestors_n, ancestors_b = Graph.fuzzy_match( - api_node_n, self.graph_b.node_map.get(recount_result_b.get(recount_node_id))) - if api_node_b: - self._process_matched_nodes(api_node_n, api_node_b, ancestors_n, ancestors_b) - for sub_node in node_n.subnodes: - self._compare_nodes_fuzzy(sub_node) + # 真实数据比对只会得到基本信息，并没有精度指标，需要调用多进程对比接口 + self._get_and_add_result(node_n, node_b) + node_list.extend(node_n.subnodes) + + node_list = [node_root] + while node_list: + compare_single_node(node_list.pop(0)) + + def _compare_nodes_fuzzy(self, node_root): + def compare_single_nodes_fuzzy(node_n): + if node_n.op != NodeOp.function_api: + # 模块经过模糊匹配 + node_b, ancestors_n, ancestors_b = Graph.fuzzy_match(node_n, self.graph_b.node_map.get(node_n.id)) + if node_b: + self._process_matched_nodes(node_n, node_b, ancestors_n, ancestors_b) + # 匹配上的两个模块中的所有api, 忽略dump调用次数，按照名称一致+模块中的调用顺序进行匹配 + recount_result_n = self._recount_api_node(node_n) + recount_result_b = self._recount_api_node(node_b) + for recount_node_id, node_id_n in recount_result_n.items(): + api_node_n = self.graph_n.node_map.get(node_id_n) + if not api_node_n: + continue + api_node_b, ancestors_n, ancestors_b = Graph.fuzzy_match( + api_node_n, self.graph_b.node_map.get(recount_result_b.get(recount_node_id))) + if api_node_b: + self._process_matched_nodes(api_node_n, api_node_b, ancestors_n, ancestors_b) + node_list.extend(node_n.subnodes) + + node_list = [node_root] + while node_list: + compare_single_nodes_fuzzy(node_list.pop(0)) def _parse_param(self, dump_path_param, output_path): self.dump_path_param = dump_path_param @@ -128,7 +136,7 @@ class GraphComparator: if not self.ma.compare_mode == GraphConst.REAL_DATA_COMPARE: return df = get_csv_df(True, self.ma.csv_data, self.ma.compare_mode) - df = run_real_data(self.dump_path_param, df, self.framework, True if self.mapping_dict else False) + df = run_real_data(self.dump_path_param, df, self.framework, self.is_cross_framework) compare_data_dict = {row[0]: row.tolist() for _, row in df.iterrows()} for node in self.ma.compare_nodes: precision_index, _ = self.ma.parse_result(node, [compare_data_dict]) diff --git a/debug/accuracy_tools/msprobe/visualization/compare/mode_adapter.py b/debug/accuracy_tools/msprobe/visualization/compare/mode_adapter.py index 7b961c4e8cdcb0b2d636d2782d3a9cce851a982f..dd6f4fb1e63106001e8f22a3cb68e0ea47cbb345 100644 --- a/debug/accuracy_tools/msprobe/visualization/compare/mode_adapter.py +++ b/debug/accuracy_tools/msprobe/visualization/compare/mode_adapter.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import math import json from msprobe.core.common.const import CompareConst, Const from msprobe.visualization.utils import ToolTip, GraphConst, str2float @@ -24,6 +25,12 @@ class ModeAdapter: self.csv_data = [] self.compare_nodes = [] + @staticmethod + def _is_invalid(value): + if not isinstance(value, float): + return False + return math.isnan(value) or math.isinf(value) + @staticmethod def _add_md5_compare_data(node_data, compare_data_dict): precision_index = GraphConst.MAX_INDEX_KEY @@ -48,6 +55,8 @@ class ModeAdapter: for key, value in node_data.items(): if not isinstance(value, dict): continue + if value.get(Const.MAX) is None: + continue compare_data = compare_data_dict.get(key) if compare_data: headers = CompareConst.COMPARE_RESULT_HEADER @@ -66,9 +75,13 @@ class ModeAdapter: if thousandth is not None: numbers.append(thousandth) node_data[key] = value + if ModeAdapter._is_invalid(value.get(Const.MAX)) or ModeAdapter._is_invalid(value.get(Const.MIN)): + numbers.append(CompareConst.N_A) # 双千指标都是None的异常情况 if not numbers: min_thousandth = None + elif CompareConst.N_A in numbers: + min_thousandth = CompareConst.N_A else: min_thousandth = min(numbers + [min_thousandth]) return min_thousandth @@ -80,6 +93,8 @@ class ModeAdapter: for key, data_info in node_data.items(): if not isinstance(data_info, dict): continue + if data_info.get(Const.MAX) is None: + continue compare_data = compare_data_dict.get(key) if compare_data: # 对应比对结果csv的列 @@ -91,6 +106,8 @@ class ModeAdapter: relative_err = str2float(data_info.get(item)) max_relative_err = max(max_relative_err, relative_err) node_data[key] = data_info + if ModeAdapter._is_invalid(data_info.get(Const.MAX)) or ModeAdapter._is_invalid(data_info.get(Const.MIN)): + max_relative_err = GraphConst.MAX_INDEX_KEY max_relative_err = 1 if max_relative_err > 1 else max_relative_err return max_relative_err @@ -132,7 +149,11 @@ class ModeAdapter: ModeAdapter._check_list_len(compare_data_dict_list, 1) min_thousandth_in = ModeAdapter._add_real_compare_data(node.input_data, compare_data_dict_list[0]) min_thousandth_out = ModeAdapter._add_real_compare_data(node.output_data, compare_data_dict_list[0]) - if min_thousandth_in is not None and min_thousandth_out is not None: + if CompareConst.N_A == min_thousandth_out: + change_percentage = GraphConst.MAX_INDEX_KEY + elif CompareConst.N_A == min_thousandth_in: + change_percentage = GraphConst.MIN_INDEX_KEY + elif min_thousandth_in is not None and min_thousandth_out is not None: change_percentage = min_thousandth_in - min_thousandth_out else: change_percentage = GraphConst.MIN_INDEX_KEY diff --git a/debug/accuracy_tools/msprobe/visualization/graph/base_node.py b/debug/accuracy_tools/msprobe/visualization/graph/base_node.py index fd1541b87bf5e7ba54a95089646683c41f546ca6..dee86180586670d6f9c0c4672375479e805f818b 100644 --- a/debug/accuracy_tools/msprobe/visualization/graph/base_node.py +++ b/debug/accuracy_tools/msprobe/visualization/graph/base_node.py @@ -87,15 +87,15 @@ class BaseNode: self.matched_node_link = ancestors node.matched_node_link = ancestors - def to_dict(self): + def to_dict(self, compare_mode=None): """ 输出数据 """ result = { 'id': self.id, 'node_type': self.op.value, - 'output_data': format_node_data(self.output_data, self.id), - 'input_data': format_node_data(self.input_data, self.id), + 'output_data': format_node_data(self.output_data, self.id, compare_mode), + 'input_data': format_node_data(self.input_data, self.id, compare_mode), 'upnode': self.upnode.id if self.upnode else 'None', 'subnodes': [node.id for node in self.subnodes], 'matched_node_link': self.matched_node_link, diff --git a/debug/accuracy_tools/msprobe/visualization/graph/graph.py b/debug/accuracy_tools/msprobe/visualization/graph/graph.py index 90574174144ecc6b53033871dceda2bc53c87ba5..5bcad6446ca29ca09a986c315690bbfe2c26d36f 100644 --- a/debug/accuracy_tools/msprobe/visualization/graph/graph.py +++ b/debug/accuracy_tools/msprobe/visualization/graph/graph.py @@ -146,7 +146,7 @@ class Graph: """ return self.node_map.get(node_id, None) - def to_dict(self): + def to_dict(self, compare_mode=None): """ 用于数据输出 """ @@ -155,7 +155,7 @@ class Graph: result[GraphConst.JSON_DATA_KEY] = self.data_path result[GraphConst.JSON_NODE_KEY] = {} for node_id in self.node_map: - info = self.node_map.get(node_id).to_dict() + info = self.node_map.get(node_id).to_dict(compare_mode) result[GraphConst.JSON_NODE_KEY][node_id] = info return result diff --git a/debug/accuracy_tools/msprobe/visualization/graph_service.py b/debug/accuracy_tools/msprobe/visualization/graph_service.py index d971320a594e20baa4fb5349c0c776ffcca9b993..b14ccab0386be92c0cdce7ebc89854a9ce17aa92 100644 --- a/debug/accuracy_tools/msprobe/visualization/graph_service.py +++ b/debug/accuracy_tools/msprobe/visualization/graph_service.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,83 +15,93 @@ import os import time -import json +from copy import deepcopy +from multiprocessing import cpu_count, Pool from msprobe.core.common.file_utils import (check_file_type, create_directory, FileChecker, check_file_or_directory_path, load_json) from msprobe.core.common.const import FileCheckConst, Const -from msprobe.core.common.utils import CompareException -from msprobe.core.overflow_check.checker import AnomalyDetector +from msprobe.core.common.utils import CompareException, get_dump_mode from msprobe.visualization.compare.graph_comparator import GraphComparator -from msprobe.visualization.utils import GraphConst, check_directory_content -from msprobe.visualization.builder.graph_builder import GraphBuilder, GraphExportConfig +from msprobe.visualization.utils import GraphConst, check_directory_content, SerializableArgs +from msprobe.visualization.builder.graph_builder import GraphBuilder, GraphExportConfig, GraphInfo, BuildGraphTaskInfo from msprobe.core.common.log import logger from msprobe.visualization.graph.node_colors import NodeColors from msprobe.core.compare.layer_mapping import generate_api_mapping_by_layer_mapping from msprobe.core.compare.utils import check_and_return_dir_contents +from msprobe.core.common.utils import detect_framework_by_dump_json from msprobe.visualization.graph.distributed_analyzer import DistributedAnalyzer current_time = time.strftime("%Y%m%d%H%M%S") -def _compare_graph(input_param, args): - logger.info('Start building model graphs...') - # 对两个数据进行构图 - dump_path_n = input_param.get('npu_path') - dump_path_b = input_param.get('bench_path') - construct_path_n = FileChecker(os.path.join(dump_path_n, GraphConst.CONSTRUCT_FILE), - FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check() - construct_path_b = FileChecker(os.path.join(dump_path_b, GraphConst.CONSTRUCT_FILE), - FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check() - data_path_n = FileChecker(os.path.join(dump_path_n, GraphConst.DUMP_FILE), FileCheckConst.FILE, - FileCheckConst.READ_ABLE).common_check() - data_path_b = FileChecker(os.path.join(dump_path_b, GraphConst.DUMP_FILE), FileCheckConst.FILE, - FileCheckConst.READ_ABLE).common_check() - stack_path_n = FileChecker(os.path.join(dump_path_n, GraphConst.STACK_FILE), FileCheckConst.FILE, - FileCheckConst.READ_ABLE).common_check() - stack_path_b = FileChecker(os.path.join(dump_path_b, GraphConst.STACK_FILE), FileCheckConst.FILE, - FileCheckConst.READ_ABLE).common_check() - graph_n = GraphBuilder.build(construct_path_n, data_path_n, stack_path_n, complete_stack=args.complete_stack) - graph_b = GraphBuilder.build(construct_path_b, data_path_b, stack_path_b, complete_stack=args.complete_stack) - logger.info('Model graphs built successfully, start Comparing graphs...') - # 基于graph、stack和data进行比较 +def _compare_graph(graph_n: GraphInfo, graph_b: GraphInfo, input_param, args): dump_path_param = { - 'npu_json_path': data_path_n, - 'bench_json_path': data_path_b, - 'stack_json_path': stack_path_n, + 'npu_json_path': graph_n.data_path, + 'bench_json_path': graph_b.data_path, + 'stack_json_path': graph_n.stack_path, 'is_print_compare_log': input_param.get("is_print_compare_log", True) } - mapping_dict = None + mapping_dict = {} if args.layer_mapping: - yaml_path = FileChecker(args.layer_mapping, FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check() try: - mapping_dict = generate_api_mapping_by_layer_mapping(data_path_n, data_path_b, yaml_path) + mapping_dict = generate_api_mapping_by_layer_mapping(graph_n.data_path, graph_b.data_path, + args.layer_mapping) except Exception: logger.warning('The layer mapping file parsing failed, please check file format, mapping is not effective.') - graph_comparator = GraphComparator([graph_n, graph_b], dump_path_param, args, mapping_dict=mapping_dict) + is_cross_framework = detect_framework_by_dump_json(graph_n.data_path) != \ + detect_framework_by_dump_json(graph_b.data_path) + if is_cross_framework and not args.layer_mapping: + logger.error('The cross_frame graph comparison failed. ' + 'Please specify -lm or --layer_mapping when performing cross_frame graph comparison.') + raise CompareException(CompareException.CROSS_FRAME_ERROR) + + graph_comparator = GraphComparator([graph_n.graph, graph_b.graph], dump_path_param, args, is_cross_framework, + mapping_dict=mapping_dict) graph_comparator.compare() - micro_steps = graph_n.paging_by_micro_step(graph_b) + return graph_comparator + + +def _compare_graph_result(input_param, args): + logger.info('Start building model graphs...') + # 对两个数据进行构图 + graph_n = _build_graph_info(input_param.get('npu_path'), args) + graph_b = _build_graph_info(input_param.get('bench_path'), args) + logger.info('Model graphs built successfully, start Comparing graphs...') + # 基于graph、stack和data进行比较 + graph_comparator = _compare_graph(graph_n, graph_b, input_param, args) + # 增加micro step标记 + micro_steps = graph_n.graph.paging_by_micro_step(graph_b.graph) # 开启溢出检测 if args.overflow_check: - graph_n.overflow_check() - graph_b.overflow_check() + graph_n.graph.overflow_check() + graph_b.graph.overflow_check() - return CompareGraphResult(graph_n, graph_b, graph_comparator, micro_steps) + return CompareGraphResult(graph_n.graph, graph_b.graph, graph_comparator, micro_steps) -def _export_compare_graph_result(args, graphs, graph_comparator, micro_steps, - output_file_name=f'compare_{current_time}.vis'): - create_directory(args.output_path) +def _export_compare_graph_result(args, result): + graphs = [result.graph_n, result.graph_b] + graph_comparator = result.graph_comparator + micro_steps = result.micro_steps + output_file_name = result.output_file_name + if not output_file_name: + output_file_name = f'compare_{current_time}.vis' + logger.info(f'Start exporting compare graph result, file name: {output_file_name}...') output_path = os.path.join(args.output_path, output_file_name) task = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(graph_comparator.ma.compare_mode) export_config = GraphExportConfig(graphs[0], graphs[1], graph_comparator.ma.get_tool_tip(), NodeColors.get_node_colors(graph_comparator.ma.compare_mode), micro_steps, task, - args.overflow_check) - GraphBuilder.to_json(output_path, export_config) - logger.info(f'Model graphs compared successfully, the result file is saved in {output_path}') + args.overflow_check, graph_comparator.ma.compare_mode) + try: + GraphBuilder.to_json(output_path, export_config) + logger.info(f'Exporting compare graph result successfully, the result file is saved in {output_path}') + return '' + except RuntimeError as e: + logger.error(f'Failed to export compare graph result, file: {output_file_name}, error: {e}') + return output_file_name -def _build_graph(dump_path, args): - logger.info('Start building model graph...') +def _build_graph_info(dump_path, args): construct_path = FileChecker(os.path.join(dump_path, GraphConst.CONSTRUCT_FILE), FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check() data_path = FileChecker(os.path.join(dump_path, GraphConst.DUMP_FILE), FileCheckConst.FILE, @@ -99,6 +109,13 @@ def _build_graph(dump_path, args): stack_path = FileChecker(os.path.join(dump_path, GraphConst.STACK_FILE), FileCheckConst.FILE, FileCheckConst.READ_ABLE).common_check() graph = GraphBuilder.build(construct_path, data_path, stack_path, complete_stack=args.complete_stack) + return GraphInfo(graph, construct_path, data_path, stack_path) + + +def _build_graph_result(dump_path, args): + logger.info('Start building model graphs...') + graph = _build_graph_info(dump_path, args).graph + # 增加micro step标记 micro_steps = graph.paging_by_micro_step() # 开启溢出检测 if args.overflow_check: @@ -106,15 +123,128 @@ def _build_graph(dump_path, args): return BuildGraphResult(graph, micro_steps) -def _export_build_graph_result(out_path, graph, micro_steps, overflow_check, - output_file_name=f'build_{current_time}.vis'): - create_directory(out_path) +def _run_build_graph_compare(input_param, args, nr, br): + logger.info(f'Start building graph for {nr}...') + graph_n = _build_graph_info(input_param.get('npu_path'), args) + graph_b = _build_graph_info(input_param.get('bench_path'), args) + logger.info(f'Building graph for {nr} finished.') + return BuildGraphTaskInfo(graph_n, graph_b, nr, br, current_time) + + +def _run_build_graph_single(dump_ranks_path, rank, step, args): + logger.info(f'Start building graph for {rank}...') + dump_path = os.path.join(dump_ranks_path, rank) + output_file_name = f'build_{step}_{rank}_{current_time}.vis' if step else f'build_{rank}_{current_time}.vis' + result = _build_graph_result(dump_path, args) + result.output_file_name = output_file_name + if rank != Const.RANK: + try: + result.rank = int(rank.replace(Const.RANK, "")) + except Exception as e: + logger.error('The folder name format is incorrect, expected rank+number.') + raise CompareException(CompareException.INVALID_PATH_ERROR) from e + logger.info(f'Building graph for step: {step}, rank: {rank} finished.') + return result + + +def _run_graph_compare(graph_task_info, input_param, args, output_file_name): + logger.info(f'Start comparing data for {graph_task_info.npu_rank}...') + graph_n = graph_task_info.graph_info_n + graph_b = graph_task_info.graph_info_b + nr = graph_task_info.npu_rank + graph_comparator = _compare_graph(graph_n, graph_b, input_param, args) + micro_steps = graph_n.graph.paging_by_micro_step(graph_b.graph) + # 开启溢出检测 + if args.overflow_check: + graph_n.graph.overflow_check() + graph_b.graph.overflow_check() + graph_result = CompareGraphResult(graph_n.graph, graph_b.graph, graph_comparator, micro_steps) + graph_result.output_file_name = output_file_name + if nr != Const.RANK: + try: + graph_result.rank = int(nr.replace(Const.RANK, "")) + except Exception as e: + logger.error('The folder name format is incorrect, expected rank+number.') + raise CompareException(CompareException.INVALID_PATH_ERROR) from e + logger.info(f'Comparing data for {graph_task_info.npu_rank} finished.') + return graph_result + + +def _export_build_graph_result(args, result): + out_path = args.output_path + graph = result.graph + micro_steps = result.micro_steps + overflow_check = args.overflow_check + output_file_name = result.output_file_name + if not output_file_name: + output_file_name = f'build_{current_time}.vis' + logger.info(f'Start exporting graph for {output_file_name}...') output_path = os.path.join(out_path, output_file_name) - GraphBuilder.to_json(output_path, GraphExportConfig(graph, micro_steps=micro_steps, overflow_check=overflow_check)) - logger.info(f'Model graph built successfully, the result file is saved in {output_path}') + try: + GraphBuilder.to_json(output_path, GraphExportConfig(graph, micro_steps=micro_steps, + overflow_check=overflow_check)) + logger.info(f'Model graph exported successfully, the result file is saved in {output_path}') + return None + except RuntimeError as e: + logger.error(f'Failed to export model graph, file: {output_file_name}, error: {e}') + return output_file_name + + +def is_real_data_compare(input_param, npu_ranks, bench_ranks): + dump_rank_n = input_param.get('npu_path') + dump_rank_b = input_param.get('bench_path') + has_real_data = False + for nr, br in zip(npu_ranks, bench_ranks): + dump_path_param = { + 'npu_json_path': FileChecker(os.path.join(dump_rank_n, nr, GraphConst.DUMP_FILE), FileCheckConst.FILE, + FileCheckConst.READ_ABLE).common_check(), + 'bench_json_path': FileChecker(os.path.join(dump_rank_b, br, GraphConst.DUMP_FILE), FileCheckConst.FILE, + FileCheckConst.READ_ABLE).common_check() + } + has_real_data |= get_dump_mode(dump_path_param) == Const.ALL + return has_real_data + + +def _mp_compare(input_param, serializable_args, output_file_name, nr, br): + graph_task_info = _run_build_graph_compare(input_param, serializable_args, nr, br) + return _run_graph_compare(graph_task_info, input_param, serializable_args, output_file_name) def _compare_graph_ranks(input_param, args, step=None): + with Pool(processes=max(int((cpu_count() + 1) // 4), 1)) as pool: + def err_call(err): + logger.error(f'Error occurred while comparing graph ranks: {err}') + try: + pool.close() + except OSError as e: + logger.error(f'Error occurred while terminating the pool: {e}') + + serializable_args = SerializableArgs(args) + # 暂存所有rank的graph，用于匹配rank间的分布式节点 + compare_graph_results = _get_compare_graph_results(input_param, serializable_args, step, pool, err_call) + + # 匹配rank间的分布式节点 + if len(compare_graph_results) > 1: + DistributedAnalyzer({obj.rank: obj.graph_n for obj in compare_graph_results}, + args.overflow_check).distributed_match() + DistributedAnalyzer({obj.rank: obj.graph_b for obj in compare_graph_results}, + args.overflow_check).distributed_match() + + export_res_task_list = [] + create_directory(args.output_path) + for result in compare_graph_results: + export_res_task_list.append(pool.apply_async(_export_compare_graph_result, + args=(serializable_args, result), + error_callback=err_call)) + export_res_list = [res.get() for res in export_res_task_list] + if any(export_res_list): + failed_names = list(filter(lambda x: x, export_res_list)) + logger.error(f'Unable to export compare graph results: {", ".join(failed_names)}.') + else: + logger.info('Successfully exported compare graph results.') + + +def _get_compare_graph_results(input_param, serializable_args, step, pool, err_call): dump_rank_n = input_param.get('npu_path') dump_rank_b = input_param.get('bench_path') npu_ranks = sorted(check_and_return_dir_contents(dump_rank_n, Const.RANK)) @@ -123,32 +253,33 @@ def _compare_graph_ranks(input_param, args, step=None): logger.error('The number of ranks in the two runs are different. Unable to match the ranks.') raise CompareException(CompareException.INVALID_PATH_ERROR) compare_graph_results = [] - for nr, br in zip(npu_ranks, bench_ranks): - logger.info(f'Start processing data for {nr}...') - input_param['npu_path'] = os.path.join(dump_rank_n, nr) - input_param['bench_path'] = os.path.join(dump_rank_b, br) - output_file_name = f'compare_{step}_{nr}_{current_time}.vis' if step else f'compare_{nr}_{current_time}.vis' - result = _compare_graph(input_param, args) - result.output_file_name = output_file_name - if nr != Const.RANK: - try: - result.rank = int(nr.replace(Const.RANK, "")) - except Exception as e: - logger.error('The folder name format is incorrect, expected rank+number.') - raise CompareException(CompareException.INVALID_PATH_ERROR) from e - # 暂存所有rank的graph，用于匹配rank间的分布式节点 - compare_graph_results.append(result) - - # 匹配rank间的分布式节点 - if len(compare_graph_results) > 1: - DistributedAnalyzer({obj.rank: obj.graph_n for obj in compare_graph_results}, - args.overflow_check).distributed_match() - DistributedAnalyzer({obj.rank: obj.graph_b for obj in compare_graph_results}, - args.overflow_check).distributed_match() - - for result in compare_graph_results: - _export_compare_graph_result(args, [result.graph_n, result.graph_b], result.graph_comparator, - result.micro_steps, output_file_name=result.output_file_name) + if is_real_data_compare(input_param, npu_ranks, bench_ranks): + mp_task_dict = {} + for nr, br in zip(npu_ranks, bench_ranks): + input_param['npu_path'] = os.path.join(dump_rank_n, nr) + input_param['bench_path'] = os.path.join(dump_rank_b, br) + output_file_name = f'compare_{step}_{nr}_{current_time}.vis' if step else f'compare_{nr}_{current_time}.vis' + input_param_copy = deepcopy(input_param) + mp_task_dict[output_file_name] = pool.apply_async(_run_build_graph_compare, + args=(input_param_copy, serializable_args, nr, br), + error_callback=err_call) + + mp_res_dict = {k: v.get() for k, v in mp_task_dict.items()} + for output_file_name, mp_res in mp_res_dict.items(): + compare_graph_results.append(_run_graph_compare(mp_res, input_param, serializable_args, output_file_name)) + else: + compare_graph_tasks = [] + for nr, br in zip(npu_ranks, bench_ranks): + input_param['npu_path'] = os.path.join(dump_rank_n, nr) + input_param['bench_path'] = os.path.join(dump_rank_b, br) + output_file_name = f'compare_{step}_{nr}_{current_time}.vis' if step else f'compare_{nr}_{current_time}.vis' + input_param_copy = deepcopy(input_param) + compare_graph_tasks.append(pool.apply_async(_mp_compare, + args=(input_param_copy, serializable_args, output_file_name, nr, + br), + error_callback=err_call)) + compare_graph_results = [task.get() for task in compare_graph_tasks] + return compare_graph_results def _compare_graph_steps(input_param, args): @@ -172,28 +303,39 @@ def _compare_graph_steps(input_param, args): def _build_graph_ranks(dump_ranks_path, args, step=None): ranks = sorted(check_and_return_dir_contents(dump_ranks_path, Const.RANK)) - build_graph_results = [] - for rank in ranks: - logger.info(f'Start processing data for {rank}...') - dump_path = os.path.join(dump_ranks_path, rank) - output_file_name = f'build_{step}_{rank}_{current_time}.vis' if step else f'build_{rank}_{current_time}.vis' - result = _build_graph(dump_path, args) - result.output_file_name = output_file_name - if rank != Const.RANK: + serializable_args = SerializableArgs(args) + with Pool(processes=max(int((cpu_count() + 1) // 4), 1)) as pool: + def err_call(err): + logger.error(f'Error occurred while comparing graph ranks: {err}') try: - result.rank = int(rank.replace(Const.RANK, "")) - except Exception as e: - logger.error('The folder name format is incorrect, expected rank+number.') - raise CompareException(CompareException.INVALID_PATH_ERROR) from e - build_graph_results.append(result) - - if len(build_graph_results) > 1: - DistributedAnalyzer({obj.rank: obj.graph for obj in build_graph_results}, - args.overflow_check).distributed_match() + pool.close() + except OSError as e: + logger.error(f'Error occurred while terminating the pool: {e}') + + build_graph_tasks = [] + for rank in ranks: + build_graph_tasks.append(pool.apply_async(_run_build_graph_single, + args=(dump_ranks_path, rank, step, serializable_args), + error_callback=err_call)) + build_graph_results = [task.get() for task in build_graph_tasks] + + if len(build_graph_results) > 1: + DistributedAnalyzer({obj.rank: obj.graph for obj in build_graph_results}, + args.overflow_check).distributed_match() + + create_directory(args.output_path) + export_build_graph_tasks = [] + for result in build_graph_results: + export_build_graph_tasks.append(pool.apply_async(_export_build_graph_result, + args=(serializable_args, result), + error_callback=err_call)) + export_build_graph_result = [task.get() for task in export_build_graph_tasks] + if any(export_build_graph_result): + failed_names = list(filter(lambda x: x, export_build_graph_result)) + logger.error(f'Unable to export build graph results: {failed_names}.') + else: + logger.info(f'Successfully exported build graph results.') - for result in build_graph_results: - _export_build_graph_result(args.output_path, result.graph, result.micro_steps, args.overflow_check, - result.output_file_name) def _build_graph_steps(dump_steps_path, args): @@ -209,7 +351,7 @@ def _graph_service_parser(parser): help=" The compare input path, a dict json.", required=True) parser.add_argument("-o", "--output_path", dest="output_path", type=str, help=" The compare task result out path.", required=True) - parser.add_argument("-lm", "--layer_mapping", dest="layer_mapping", type=str, + parser.add_argument("-lm", "--layer_mapping", dest="layer_mapping", type=str, nargs='?', const=True, help=" The layer mapping file path.", required=False) parser.add_argument("-oc", "--overflow_check", dest="overflow_check", action="store_true", help=" whether open overflow_check for graph.", required=False) @@ -233,8 +375,11 @@ def _graph_service_command(args): elif content == GraphConst.STEPS: _build_graph_steps(npu_path, args) else: - result = _build_graph(npu_path, args) - _export_build_graph_result(args.output_path, result.graph, result.micro_steps, args.overflow_check) + result = _build_graph_result(npu_path, args) + create_directory(args.output_path) + file_name = _export_build_graph_result(args, result) + if file_name: + logger.error('Failed to export model build graph.') elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: content_n = check_directory_content(npu_path) content_b = check_directory_content(bench_path) @@ -245,9 +390,11 @@ def _graph_service_command(args): elif content_n == GraphConst.STEPS: _compare_graph_steps(input_param, args) else: - result = _compare_graph(input_param, args) - _export_compare_graph_result(args, [result.graph_n, result.graph_b], - result.graph_comparator, result.micro_steps) + result = _compare_graph_result(input_param, args) + create_directory(args.output_path) + file_name = _export_compare_graph_result(args, result) + if file_name: + logger.error('Failed to export model compare graph.') else: logger.error("The npu_path or bench_path should be a folder.") raise CompareException(CompareException.INVALID_COMPARE_MODE) diff --git a/debug/accuracy_tools/msprobe/visualization/utils.py b/debug/accuracy_tools/msprobe/visualization/utils.py index b4ea6b29596276f3587e8e010ab178b12eb3c4e7..242d641e31ae54c99a347f29928bca38523fa975 100644 --- a/debug/accuracy_tools/msprobe/visualization/utils.py +++ b/debug/accuracy_tools/msprobe/visualization/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,9 +16,10 @@ import os import re import json +import pickle from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.const import CompareConst, Const -from msprobe.core.compare.acc_compare import Comparator, ModeConfig +from msprobe.core.common.log import logger def load_json_file(file_path): @@ -42,15 +43,6 @@ def load_data_json_file(file_path): return load_json_file(file_path).get(GraphConst.DATA_KEY, {}) -def get_csv_df(stack_mode, csv_data, compare_mode): - """ - 调用acc接口写入csv - """ - dump_mode = GraphConst.GRAPHCOMPARE_MODE_TO_DUMP_MODE_TO_MAPPING.get(compare_mode) - mode_config = ModeConfig(stack_mode=stack_mode, dump_mode=dump_mode) - return Comparator(mode_config).make_result_table(csv_data) - - def str2float(percentage_str): """ 百分比字符串转换转换为浮点型 @@ -127,14 +119,12 @@ class ToolTip: '当最大相对误差越接近0表示其计算的误差越小。' '当dump数据中存在0或Nan时，比对结果中最大相对误差则出现inf或Nan的情况，属于正常现象' ) - SMALL_VALUE_TIP = '{}, 由于{}小于{}, 建议不参考此相对误差，请参考绝对误差' class GraphConst: CONSTRUCT_FILE = 'construct.json' DUMP_FILE = 'dump.json' STACK_FILE = 'stack.json' - GRAPH_FILE = 'graph.vis' ERROR_KEY = 'error_key' SUMMARY_COMPARE = 0 MD5_COMPARE = 1 @@ -148,32 +138,22 @@ class GraphConst: JSON_DATA_KEY = 'dump_data_dir' JSON_TASK_KEY = 'task' DATA_KEY = 'data' - REAL_DATA_TH = 0.1 - MAX_RELATIVE_ERR_TH = 0.5 ROUND_TH = 6 JSON_INDEX_KEY = 'precision_index' MATCHED_DISTRIBUTED = 'matched_distributed' OVERFLOW_LEVEL = 'overflow_level' MAX_INDEX_KEY = 1 MIN_INDEX_KEY = 0 - SUGGEST_KEY = 'text' - TAG_NA = 'na' - OUTPUT_INDEX_TWO = -2 - OUTPUT_INDEX_THREE = -3 - OUTPUT_MIN_LEN = 3 INPUT = '.input.' OUTPUT = '.output.' STR_MAX_LEN = 50 - SMALL_VALUE = 1e-3 MD5_INDEX_LIST = [CompareConst.RESULT] REAL_DATA_INDEX_LIST = CompareConst.ALL_COMPARE_INDEX SUMMARY_INDEX_LIST = CompareConst.SUMMARY_COMPARE_INDEX - VALUE_INDEX_LIST = [Const.MAX, Const.MIN, Const.MEAN, Const.NORM] APIS_BETWEEN_MODULES = 'Apis_Between_Modules' NULL = 'null' NONE = 'None' VALUE = 'value' - BRACE = '{}' DESCRIPTION = 'description' COLORS = 'Colors' MICRO_STEPS = 'MicroSteps' @@ -204,3 +184,24 @@ class GraphConst: OP = 'op' PEER = 'peer' GROUP_ID = 'group_id' + + +def is_serializable(obj): + """ + Check if an object is serializable + """ + try: + pickle.dumps(obj) + return True + except (pickle.PicklingError, AttributeError, TypeError): + return False + except Exception as e: + logger.error('Unexpected error occurred while pickling obj.') + raise RuntimeError('Unexpected error occurred while pickling obj.') from e + + +class SerializableArgs: + def __init__(self, args): + for k, v in vars(args).items(): + if is_serializable(v): + setattr(self, k, v) diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index 14fd15e3c06deef1d0e3b9ff26b199b02f6ce391..c1e6e4cbedd6834503bfa2b87d1df080b1a4efaf 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -14,7 +14,7 @@ # limitations under the License. -__version__ = '1.2.2' +__version__ = '1.3.0' import subprocess import platform diff --git a/debug/resources/training_process.png b/debug/resources/training_process.png new file mode 100644 index 0000000000000000000000000000000000000000..e1cf2f20471624cd86edbf45444bb431086d6065 Binary files /dev/null and b/debug/resources/training_process.png differ diff --git a/dynolog_npu/README.md b/dynolog_npu/README.md deleted file mode 100644 index 86a23b7f82925079c26623b070936538768d9b8c..0000000000000000000000000000000000000000 --- a/dynolog_npu/README.md +++ /dev/null @@ -1,221 +0,0 @@ -# Ascend Extension for dynolog - -## 安装方式 - -### 1. clone 代码 - -```bash -git clone https://gitee.com/ascend/mstt.git -``` - -### 2. 安装依赖 -dynolog的编译依赖，确保安装了以下依赖： -
采集模式global_batch_size单卡8卡
L0113GB97GB
225B194GB
337G291GB
225GB194GB
337GB291GB
L11440GB3.4TB
2720GB5.4TB
3960GB7.3TB
- - - - - - - - - - - - -
Language - Toolchain -
C++ - gcc 8.5.0+ -
Rust - Rust 1.58.1 (1.56+ required for clap dependency) -
- -- 安装rust - -```bash -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - -source $HOME/.cargo/env -``` - -- 安装ninja - -```bash -# debian -sudo apt-get install -y cmake ninja-build - -# centos -sudo yum install -y cmake ninja -``` - -### 3. 编译 - -- dynolog编译 - -默认编译生成dyno和dynolog二进制文件, -t参数可以支持将二进制文件打包成deb包或rpm包. - -```bash -# 编译dyno和dynolog二进制文件 -bash scripts/build.sh - -# 编译deb包, 当前支持amd64和aarch64平台, 默认为amd64, 编译aarch64平台需要修改third_party/dynolog/scripts/debian/control文件中的Architecture改为aarch64 -bash scripts/build.sh -t deb - -# 编译rpm包, 当前只支持amd64平台 -bash scripts/build.sh -t rpm -``` - -- dynolog_npu_plugin wheel包编译 - -dynolog_npu_plugin wheel包提供IPCMonitor，MsptiMonitor等公共能力，使用nputrace和npu-monitor功能前必须安装该wheel包，具体编译安装指导可参考dynolog_npu\plugin\README.md。 - -## 使用方式 - -### Profiler trace dump功能 -Profiler trace dump功能基于dynolog开发，实现类似于动态profiling的动态触发Ascend Torch Profiler采集profiling的功能。用户基于dyno CLI命令行可以动态触发指定节点的训练进程trace dump。 - -- 查看nputrace支持的命令和帮助 - -```bash -dyno nputrace --help -``` - -- nputrace使用方式 - -```bash -dyno nputrace [SUBCOMMANDS] --log-file -``` - -nputrace子命令支持的参数选项 - -| 子命令 | 参数类型 | 说明 | -|-------|-------|-------| -| job-id | u64 | 采集任务的job id，默认值0，dynolog原生参数 | -| pids | String | 采集任务的pid列表，多个pid用逗号分隔，默认值0，dynolog原生参数 | -| process-limit | u64 | 最大采集进程的数量，默认值3，dynolog原生参数 | -| profile-start-time | u64 | 用于同步采集的Unix时间戳，单位毫秒，默认值0，dynolog原生参数 | -| duration-ms | u64 | 采集的周期，单位毫秒，默认值500，dynolog原生参数 | -| iterations | i64 | 采集总迭代数，默认值-1，dynolog原生参数 | -| log-file | String | 采集落盘的路径，必选值 | -| start-step | u64 | 开始采集的迭代数，默认值0 | -| record-shapes | action | 是否采集算子的InputShapes和InputTypes，设置参数采集，默认不采集 | -| profile-memory | action | 是否采集算子内存信息，设置参数采集，默认不采集 | -| with-stack | action | 是否采集Python调用栈，设置参数采集，默认不采集 | -| with-flops | action | 是否采集算子flops，设置参数采集，默认不采集 | -| with-modules | action | 是否采集modules层级的Python调用栈，设置参数采集，默认不采集 | -| analyse | action | 采集后是否自动解析，设置参数解析，默认不解析 | -| l2-cache | action | 是否采集L2 Cache数据，设置参数采集，默认不采集 | -| op-attr | action | 是否采集算子属性信息，设置参数采集，默认不采集 | -| msprof-tx | action | 是否使能MSTX，设置参数采集，默认使能 | -| data-simplification | String | 解析完成后是否数据精简，可选值范围[`true`, `false`]，默认值`true` | -| activities | String | 控制CPU、NPU事件采集范围，可选值范围[`CPU,NPU`, `NPU,CPU`, `CPU`, `NPU`]，默认值`CPU,NPU` | -| profiler-level | String | 控制profiler的采集等级，可选值范围[`Level_none`, `Level0`, `Level1`, `Level2`]，默认值`Level0`| -| aic-metrics | String | AI Core的性能指标采集项，可选值范围[`AiCoreNone`, `PipeUtilization`, `ArithmeticUtilization`, `Memory`, `MemoryL0`, `ResourceConflictRatio`, `MemoryUB`, `L2Cache`, `MemoryAccess`]，默认值`AiCoreNone`| -| export-type | String | profiler解析导出数据的类型，可选值范围[`Text`, `Db`]，默认值`Text`| -| gc-detect-threshold | Option | GC检测阈值，单位ms，只采集超过阈值的GC事件。该参数为可选参数，默认不设置时不开启GC检测 | - - -- nputrace使用方法 - -Step0: 参考`3.编译`章节完成dynolog的编译，以及dynolog_npu_plugin wheel包的编译和安装。 - -Step1：拉起dynolog daemon进程 -```bash -# 方法1：使用systemd拉起service -# 修改配置文件/etc/dynolog.gflags, 使能ipc_monitor -echo "--enable_ipc_monitor" | sudo tee -a /etc/dynolog.gflags -sudo systemctl start dynolog - -# 方法2：命令行执行 -dynolog --enable-ipc-monitor - -#dynolog daemon的日志路径为：/var/log/dynolog.log -``` - -Step 2：使能dynolog trace dump环境变量 -```bash -export KINETO_USE_DAEMON=1 -``` - -Step 3: 拉起训练任务 -```bash -# 训练任务中需要使用pytorch的优化器/继承原生优化器 -bash train.sh -``` - -Step 4：使用dyno CLI动态触发trace dump -```bash -# 示例1：从第10个step开始采集，采集2个step，采集框架、CANN和device数据，同时采集完后自动解析以及解析完成不做数据精简，落盘路径为/tmp/profile_data -dyno nputrace --start-step 10 --iterations 2 --activities CPU,NPU --analyse --data-simplification false --log-file /tmp/profile_data - -# 示例2：从第10个step开始采集，采集2个step，只采集CANN和device数据，同时采集完后自动解析以及解析完成后开启数据精简，落盘路径为/tmp/profile_data -dyno nputrace --start-step 10 --iterations 2 --activities NPU --analyse --data-simplification true --log-file /tmp/profile_data - -# 示例3：从第10个step开始采集，采集2个step，只采集CANN和device数据，只采集不解析，落盘路径为/tmp/profile_data -dyno nputrace --start-step 10 --iterations 2 --activities NPU --log-file /tmp/profile_data -``` - -### NPU Monitor功能 -NPU Monitor基于MSPTI/MSTX能力开发，实现了轻量级在线监控能力，能够用于性能问题的初步定位。 - -```bash -dyno npu-monitor --help -``` - -- npu-monitor使用方式 - -```bash -dyno npu-monitor [SUBCOMMANDS] -``` - -npu-monitor子命令支持的参数选项 -| 子命令 | 参数类型 | 说明 | -|-------|-------|-------| -| npu-monitor-start | action | 开启性能监控，设置参数开启，默认不采集 | -| npu-monitor-stop | action | 停止性能监控，设置参数开启，默认不采集 | -| report-interval-s | int | 性能监控数据上报周期，单位s，需要在启动时设置。默认值60 | -| mspti-activity-kind | String | 性能监控数据上报数据类型，可以设置单个或多个，多个类型以逗号分隔，需要在启动时设置。可选值范围[`Marker`, `Kernel`, `API`, `Hccl`, `Memory`, `MemSet`, `MemCpy`] , 默认值`Marker`| - -- npu-monitor使用方法 - -Step1： 拉起dynolog daemon进程 -```bash -# 方法1：使用systemd拉起service -# 修改配置文件/etc/dynolog.gflags, 使能ipc_monitor -echo "--enable_ipc_monitor" | sudo tee -a /etc/dynolog.gflags -sudo systemctl start dynolog - -# 方法2：命令行执行 -dynolog --enable-ipc-monitor - -#dynolog daemon的日志路径为：/var/log/dynolog.log -``` - -Step 2：使能dynolog trace dump环境变量 -```bash -export KINETO_USE_DAEMON=1 -``` - -Step 3: 拉起训练任务 -```bash -# 训练任务中需要使用pytorch的优化器/继承原生优化器 -bash train.sh -``` - -Step 4：使用dyno CLI使能npu-monitor -```bash -# 示例1：开启性能监控，使用默认配置 -dyno npu-monitor --npu-monitor-start - -# 示例2：暂停性能监控 -dyno npu-monitor --npu-monitor-stop - -# 示例3：性能监控过程中修改配置 -# 上报周期30s, 上报数据类型Marker和Kernel -dyno npu-monitor --report-interval-s 30 --mspti-activity-kind Marker,Kernel - -# 示例4：性能监控开启时修改配置 -# 上报周期30s, 上报数据类型Marker和Kernel -dyno npu-monitor --npu-monitor-start --report-interval-s 30 --mspti-activity-kind Marker,Kernel -``` \ No newline at end of file diff --git a/dynolog_npu/plugin/bindings.cpp b/dynolog_npu/plugin/bindings.cpp deleted file mode 100644 index c0cdaa4d577b3a76ec2d6f3eae4b426556a56532..0000000000000000000000000000000000000000 --- a/dynolog_npu/plugin/bindings.cpp +++ /dev/null @@ -1,11 +0,0 @@ -#include -#include "ipc_monitor/PyDynamicMonitorProxy.h" - -namespace py = pybind11; - -PYBIND11_MODULE(IPCMonitor, m) { - py::class_(m, "PyDynamicMonitorProxy") - .def(py::init<>()) - .def("init_dyno", &dynolog_npu::ipc_monitor::PyDynamicMonitorProxy::InitDyno, py::arg("npuId")) - .def("poll_dyno", &dynolog_npu::ipc_monitor::PyDynamicMonitorProxy::PollDyno); -} \ No newline at end of file diff --git a/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.cpp b/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.cpp deleted file mode 100644 index bba66d7297af1eec929a0149b0b2d1df35eaf843..0000000000000000000000000000000000000000 --- a/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "DynoLogNpuMonitor.h" -#include "utils.h" - -namespace dynolog_npu { -namespace ipc_monitor { - -bool DynoLogNpuMonitor::Init() -{ - if (isInitialized_) { - LOG(ERROR) << "DynoLog npu monitor already initialized"; - return true; - } - bool res = ipcClient_.RegisterInstance(npuId_); - if (res) { - isInitialized_ = true; - LOG(INFO) << "DynoLog npu monitor initialized success!"; - } - return res; -} - -std::string DynoLogNpuMonitor::Poll() -{ - std::string res = ipcClient_.IpcClientNpuConfig(); - return res; -} - -} // namespace ipc_monitor -} // namespace dynolog_npu \ No newline at end of file diff --git a/dynolog_npu/third_party/dynolog b/dynolog_npu/third_party/dynolog deleted file mode 160000 index d5d37bc182bc2aa8fa60ba7d5ee897bacb5cbd4b..0000000000000000000000000000000000000000 --- a/dynolog_npu/third_party/dynolog +++ /dev/null @@ -1 +0,0 @@ -Subproject commit d5d37bc182bc2aa8fa60ba7d5ee897bacb5cbd4b diff --git a/msmonitor/README.md b/msmonitor/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2e9ca58ab74e63455b4ad2618b43b171c31b401e --- /dev/null +++ b/msmonitor/README.md @@ -0,0 +1,303 @@ +# msMonitor: MindStudio一站式在线监控工具 + +## 安装方式 + +### 1. clone 代码 + +```bash +git clone https://gitee.com/ascend/mstt.git +``` + +### 2. 安装依赖 +dynolog的编译依赖，确保安装了以下依赖： + + + + + + + + + + + + + +
Language + Toolchain +
C++ + gcc 8.5.0+ +
Rust + Rust >= 1.81 +
+ +- 安装rust + +```bash +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + +source $HOME/.cargo/env +``` + +- 安装ninja + +```bash +# debian +sudo apt-get install -y cmake ninja-build + +# centos +sudo yum install -y cmake ninja +``` + +- 安装openssl（RPC TLS认证）& 生成证书密钥 +安装 +```bash +# debian +sudo apt-get install -y openssl + +# centos +sudo yum install -y openssl +``` +dyno CLI与dynolog daemon之间的RPC通信使用TLS证书密钥加密，在启动dyno和dynlog二进制时需要指定证书密钥存放的路径，路径下需要满足如下结构和名称。 +**用户应使用与自己需求相符的密钥生成和存储机制，并保证密钥安全性与机密性。** +服务端证书目录结构： +```bash +server_certs +├── ca.crt +├── server.crt +└── server.key +``` +客户端证书目录结构： +```bash +client_certs +├── ca.crt +├── client.crt +└── client.key +``` + +### 3. 编译 + +- dynolog编译 + +默认编译生成dyno和dynolog二进制文件, -t参数可以支持将二进制文件打包成deb包或rpm包。 + +```bash +# 编译dyno和dynolog二进制文件 +bash scripts/build.sh + +# 编译deb包, 当前支持amd64和aarch64平台, 默认为amd64, 编译aarch64平台需要修改third_party/dynolog/scripts/debian/control文件中的Architecture改为arm64 +bash scripts/build.sh -t deb + +# 编译rpm包, 当前只支持amd64平台 +bash scripts/build.sh -t rpm +``` + +- msmonitor-plugin wheel包编译 + +msmonitor-plugin wheel包提供IPCMonitor，MsptiMonitor等公共能力，使用nputrace和npu-monitor功能前必须安装该wheel包，具体编译安装指导可参考[msmonitor-plugin编包指导](./plugin/README.md)。 + +## 使用方式 + +- **说明**：**Profiler trace dump**功能和**NPU Monitor**功能**不能**同时开启。 + +### Profiler trace dump功能 +Profiler trace dump功能基于dynolog开发，实现类似于动态profiling的动态触发Ascend Pytorch Profiler采集profiling的功能。用户基于dyno CLI命令行可以动态触发指定节点的训练进程trace dump。 + +- 查看dyno支持的命令和帮助 + +```bash +dyno --help +``` + +dyno命令支持的参数选项 + +| 命令 | 参数类型 | 说明 | +|-----------|--------|-------------------------------------| +| hostname | String | 网络中唯一标识一台设备的名称，默认值localhost | +| port | i32 | 用于区分同一设备上的不同网络服务或应用程序，默认值1778 | +| certs-dir | String | 用于指定dyno与dynolog RPC通信时TLS证书的路径，必选值 | + +- 查看nputrace支持的命令和帮助 + +```bash +dyno nputrace --help +``` + +- nputrace使用方式 + +```bash +dyno --certs-dir nputrace [SUBCOMMANDS] --log-file +``` + +nputrace子命令支持的参数选项 + +| 子命令 | 参数类型 | 说明 | PyTorch支持 | MindSpore支持 | +|---------------------|-------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------:|:-----------:| +| job-id | u64 | 采集任务的job id，默认值0，dynolog原生参数 | N | N | +| pids | String | 采集任务的pid列表，多个pid用逗号分隔，默认值0，dynolog原生参数 | N | N | +| process-limit | u64 | 最大采集进程的数量，默认值3，dynolog原生参数 | N | N | +| profile-start-time | u64 | 用于同步采集的Unix时间戳，单位毫秒，默认值0，dynolog原生参数 | N | N | +| duration-ms | u64 | 采集的周期，单位毫秒，默认值500，dynolog原生参数 | N | N | +| iterations | i64 | 采集总迭代数，默认值-1，dynolog原生参数，必选参数，需与start-step参数同时指定 | Y | Y | +| log-file | String | 采集落盘的路径，必选值 | Y | Y | +| start-step | u64 | 开始采集的迭代数，默认值0 | Y | Y | +| record-shapes | action | 是否采集算子的InputShapes和InputTypes，设置参数采集，默认不采集 | Y | Y | +| profile-memory | action | 是否采集算子内存信息，设置参数采集，默认不采集 | Y | Y | +| with-stack | action | 是否采集Python调用栈，设置参数采集，默认不采集 | Y | Y | +| with-flops | action | 是否采集算子flops，设置参数采集，默认不采集 | Y | N | +| with-modules | action | 是否采集modules层级的Python调用栈，设置参数采集，默认不采集 | Y | N | +| analyse | action | 采集后是否自动解析，设置参数解析，默认不解析 | Y | Y | +| l2-cache | action | 是否采集L2 Cache数据，设置参数采集，默认不采集 | Y | Y | +| op-attr | action | 是否采集算子属性信息，设置参数采集，默认不采集 | Y | N | +| msprof-tx | action | 是否使能MSTX，设置参数采集，默认不使能 | Y | Y | +| mstx-domain-include | Option | 使能msproftx采集mstx打点数据的情况下，配置该开关，设置实际采集的domain范围。该参数为可选参数，默认不使能 | Y | Y | +| mstx-domain-exclude | Option | 使能msproftx采集mstx打点数据的情况下，配置该开关，设置实际不采集的domain范围。该参数为可选参数，默认不使能 | Y | Y | +| data-simplification | String | 解析完成后是否数据精简，可选值范围[`true`, `false`]，默认值`true` | Y | Y | +| activities | String | 控制CPU、NPU事件采集范围，可选值范围[`CPU,NPU`, `NPU,CPU`, `CPU`, `NPU`]，默认值`CPU,NPU` | Y | Y | +| profiler-level | String | 控制profiler的采集等级，可选值范围[`Level_none`, `Level0`, `Level1`, `Level2`]，默认值`Level0` | Y | Y | +| aic-metrics | String | AI Core的性能指标采集项，可选值范围[`AiCoreNone`, `PipeUtilization`, `ArithmeticUtilization`, `Memory`, `MemoryL0`, `ResourceConflictRatio`, `MemoryUB`, `L2Cache`, `MemoryAccess`]，默认值`AiCoreNone` | Y | Y | +| export-type | String | profiler解析导出数据的类型，可选值范围[`Text`, `Db`]，默认值`Text` | Y | Y | +| gc-detect-threshold | Option | GC检测阈值，单位ms，只采集超过阈值的GC事件。该参数为可选参数，默认不设置时不开启GC检测 | Y | N | +| host-sys | String | 采集[host侧系统数据](https://www.hiascend.com/document/detail/zh/mindstudio/70RC3/T&ITools/Profiling/atlasprofiling_16_0014.html)(CPU利用率、内存利用率、磁盘I/O利用率、网络I/O利用率等)。该参数为可选参数，可选值范围[`cpu`, `mem`, `disk`, `network`, `osrt`] , 默认不设置时不开启host侧系统数据采集 | Y | Y | +| sys-io | action | 采集NIC、ROCE数据。该参数为可选参数，默认不设置时不开启NIC、ROCE数据采集 | Y | Y | +| sys-interconnection | action | 采集集合通信带宽数据（HCCS）、PCIe、片间传输带宽数据。该参数为可选参数，默认不设置时不开启HCCS、PCIe、片间传输带宽数据采集 | Y | Y | + +- nputrace使用方法 + +Step 0: 参考[3.编译](./README.md#3-编译)章节完成dynolog的编译，以及dynolog_npu_plugin wheel包的编译和安装。 + +Step 1：拉起dynolog daemon进程 +```bash +# 方法1和方法2 二选一 +# 方法1：使用systemd拉起service +# 修改配置文件/etc/dynolog.gflags, 使能ipc_monitor +echo "--enable_ipc_monitor" | sudo tee -a /etc/dynolog.gflags +sudo systemctl start dynolog + +# 方法2：命令行执行 +dynolog --enable-ipc-monitor --certs-dir /home/server_certs + +#dynolog daemon的日志路径为：/var/log/dynolog.log +``` + +Step 2：在训练任务拉起窗口使能dynolog环境变量 +```bash +export KINETO_USE_DAEMON=1 +``` + +Step 3: 拉起训练任务 +```bash +# 训练任务中需要使用pytorch的优化器/继承原生优化器 +bash train.sh +``` + +Step 4：使用dyno CLI动态触发trace dump +```bash +# 示例1：从第10个step开始采集，采集2个step，采集框架、CANN和device数据，同时采集完后自动解析以及解析完成不做数据精简，落盘路径为/tmp/profile_data +dyno --certs-dir /home/client_certs nputrace --start-step 10 --iterations 2 --activities CPU,NPU --analyse --data-simplification false --log-file /tmp/profile_data + +# 示例2：从第10个step开始采集，采集2个step，只采集CANN和device数据，同时采集完后自动解析以及解析完成后开启数据精简，落盘路径为/tmp/profile_data +dyno --certs-dir /home/client_certs nputrace --start-step 10 --iterations 2 --activities NPU --analyse --data-simplification true --log-file /tmp/profile_data + +# 示例3：从第10个step开始采集，采集2个step，只采集CANN和device数据，只采集不解析，落盘路径为/tmp/profile_data +dyno --certs-dir /home/client_certs nputrace --start-step 10 --iterations 2 --activities NPU --log-file /tmp/profile_data + +# 示例4：多机场景下向特定机器x.x.x.x发送参数信息，参数表示从第10个step开始采集，采集2个step，只采集CANN和device数据，只采集不解析，落盘路径为/tmp/profile_data +dyno --certs-dir /home/client_certs --hostname x.x.x.x nputrace --start-step 10 --iterations 2 --activities NPU --log-file /tmp/profile_data +``` + +### NPU Monitor功能 +NPU Monitor基于MSPTI/MSTX能力开发，实现了轻量级在线监控能力，能够用于性能问题的初步定位。 + +**注意**：NPU Monitor功能开启时，不能同时开启Profiler trace dump功能。 + +```bash +dyno npu-monitor --help +``` + +- npu-monitor使用方式 + +```bash +dyno --certs-dir npu-monitor [SUBCOMMANDS] +``` + +npu-monitor子命令支持的参数选项 + +| 子命令 | 参数类型 | 说明 | PyTorch支持 | MindSpore支持 | +|-------|-------|----------------------------------------------------------------------------------------------------------------------------------|:---------:|:-----------:| +| npu-monitor-start | action | 开启性能监控，设置参数后生效，默认不生效 | Y | Y | +| npu-monitor-stop | action | 停止性能监控，设置参数后生效，默认不生效 | Y | Y | +| report-interval-s | int | 性能监控数据上报周期，单位s，需要在启动时设置。默认值60 | Y | Y | +| mspti-activity-kind | String | 性能监控数据上报数据类型，可以设置单个或多个，多个类型以逗号分隔，每次设置时刷新全局上报类型。可选值范围[`Marker`, `Kernel`, `API`, `Hccl`, `Memory`, `MemSet`, `MemCpy`] , 默认值`Marker` | Y | Y | + +- npu-monitor使用方法 + +Step 1： 拉起dynolog daemon进程 +```bash +# 方法1和方法2 二选一 +# 方法1：使用systemd拉起service +# 修改配置文件/etc/dynolog.gflags, 使能ipc_monitor +echo "--enable_ipc_monitor" | sudo tee -a /etc/dynolog.gflags +sudo systemctl start dynolog + +# 方法2：命令行执行 +dynolog --enable-ipc-monitor --certs-dir /home/server_certs + +# 使用Prometheus上报数据需要指定参数：--use_prometheus +# dynolog daemon的日志路径为：/var/log/dynolog.log +``` + +Step 2：在训练任务拉起窗口使能dynolog环境变量 +```bash +export KINETO_USE_DAEMON=1 +``` + +Step 3: 配置Msmonitor日志路径(可选，默认路径为当前目录下的msmonitor_log) +```bash +export MSMONITOR_LOG_PATH= +# 示例： +export MSMONITOR_LOG_PATH=/tmp/msmonitor_log +``` + +Step 4: 拉起训练任务 +```bash +# 训练任务拉起前需要设置LD_PRELOAD +# 示例：export LD_PRELOAD=/usr/local/Ascend/ascend-toolkit/latest/lib64/libmspti.so +export LD_PRELOAD=/ascend-toolkit/latest/lib64/libmspti.so + +# 训练任务中需要使用pytorch的优化器/继承原生优化器 +bash train.sh +``` + +Step 5：使用dyno CLI使能npu-monitor +```bash +# 示例1：开启性能监控，使用默认配置 +dyno --certs-dir /home/client_certs npu-monitor --npu-monitor-start + +# 示例2：暂停性能监控 +dyno --certs-dir /home/client_certs npu-monitor --npu-monitor-stop + +# 示例3：性能监控过程中修改配置 +# 上报周期30s, 上报数据类型Marker和Kernel +dyno --certs-dir /home/client_certs npu-monitor --report-interval-s 30 --mspti-activity-kind Marker,Kernel + +# 示例4：性能监控开启时修改配置 +# 上报周期30s, 上报数据类型Marker和Kernel +dyno --certs-dir /home/client_certs npu-monitor --npu-monitor-start --report-interval-s 30 --mspti-activity-kind Marker,Kernel + +# 示例5：多机场景下性能监控开启时修改配置 +# 多机场景下向特定机器x.x.x.x发送参数信息，参数表示上报周期30s, 上报数据类型Marker和Kernel +dyno --certs-dir /home/client_certs --hostname x.x.x.x npu-monitor --npu-monitor-start --report-interval-s 30 --mspti-activity-kind Marker,Kernel +``` + +Step6: 观测Prometheus上报数据 +``` +# Prometheus默认端口为8080 +curl 127.0.0.1:8080/metrics +``` + +## 附录 + +[Mindspore框架下msMonitor的使用方法](./docs/mindspore_adapter.md) + +[安全声明](./docs/security_statement.md) \ No newline at end of file diff --git a/msmonitor/docs/mindspore_adapter.md b/msmonitor/docs/mindspore_adapter.md new file mode 100644 index 0000000000000000000000000000000000000000..cb048e81dc766cc2c6156dafdb73ee30702fb853 --- /dev/null +++ b/msmonitor/docs/mindspore_adapter.md @@ -0,0 +1,60 @@ +## MindSpore框架下msMonitor的使用方法 + +### 1. 动态profiling自定义for循环方式 + +Step 1：拉起dynolog daemon进程 + +Step 2：使能dynolog环境变量 + +Step 3：配置msMonitor日志路径 + +- 前3步以及第5步操作可以参考[msMonitor使用教程](/msmonitor/README.md) + +Step 4: 拉起训练任务 +在训练任务中实例化DynamicProfilerMonitor对象，且在每一次训练后，调用step()方法。 + +- 示例代码如下： +```python +import numpy as np +import mindspore +import mindspore.dataset as ds +from mindspore import nn +from mindspore.profiler import DynamicProfilerMonitor + +class Net(nn.Cell): + def __init__(self): + super(Net, self).__init__() + self.fc = nn.Dense(2, 2) + + def construct(self, x): + return self.fc(x) + + +def generator_net(): + for _ in range(2): + yield np.ones([2, 2]).astype(np.float32), np.ones([2]).astype(np.int32) + + +def train(test_net): + optimizer = nn.Momentum(test_net.trainable_params(), 1, 0.9) + loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True) + data = ds.GeneratorDataset(generator_net(), ["data", "label"]) + model = mindspore.train.Model(test_net, loss, optimizer) + model.train(1, data) + +if __name__ == '__main__': + dp = DynamicProfilerMonitor() + step_num = 100 + # 定义模型 + net = Net() + for i in range(step_num): + # 模型训练 + train(net) + # 调用step方法实现npu trace dump或npu monitor功能 + dp.step() +``` + +Step 5：使用dyno CLI使能trace dump或npu-monitor + +### 2. 动态profiling call back方式 +该使能方式与动态profiling自定义for循环方式一致，唯一区别是将step()方法适配在step_begin、step_end回调函数中。 diff --git a/msmonitor/docs/security_statement.md b/msmonitor/docs/security_statement.md new file mode 100644 index 0000000000000000000000000000000000000000..fafd0d5ba4fa944fcecb0d8383a9058ae4e77997 --- /dev/null +++ b/msmonitor/docs/security_statement.md @@ -0,0 +1,6 @@ +## 安全声明 +### 通信矩阵 + +| 序号 | 代码仓 | 功能 | 源设备 | 源IP | 源端口 | 目的设备 | 目的IP | 目的端口
（侦听） | 协议 | 端口说明 | 端口配置 | 侦听端口是否可更改 | 认证方式 | 加密方式 | 所属平面 | 版本 | 特殊场景 | 备注 | +|:----|:------------|:-----------|:------------------|:---------------------|:------|:-------------------|:---------------------|:--------------|:-----------|:-------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------|:-----|:-----|:-------|:-----------------------|:-----|:---| +| 1 | msMonitor | dyno和dynolog RPC通信 | dyno客户端 | 运行dyno客户端进程的服务器的ip | | dynolog服务端所在服务器 | dynolog服务端所在服务器的ip | 1778 | TCP | RPC通信 | 不涉及 | 可修改 | 证书密钥 | TLS | 业务面 | 所有版本 | 无 | | diff --git a/msmonitor/dynolog_npu/cli/Cargo.toml b/msmonitor/dynolog_npu/cli/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..7d87551ba4f2e9dc3b6710ab44964365e41910e3 --- /dev/null +++ b/msmonitor/dynolog_npu/cli/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "dyno" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0.57" +clap = { version = "3.1.0", features = ["derive"]} +serde_json = "1.0" +rustls = "0.21.0" +rustls-pemfile = "1.0" +webpki = "0.22" + +[net] +git-fetch-with-cli = true + +[build] +rustflags = [ + "-C", "relocation_model=pie", + "-C", "link-args=-Wl,-z,now", + "-C", "link-args=-Wl,-z,relro", + "-C", "strip=symbols", + "-C", "overflow_checks" +] \ No newline at end of file diff --git a/msmonitor/dynolog_npu/cli/src/commands/dcgm.rs b/msmonitor/dynolog_npu/cli/src/commands/dcgm.rs new file mode 100644 index 0000000000000000000000000000000000000000..a5261fc8acefe0199340b9d7ca77903a533ee3d7 --- /dev/null +++ b/msmonitor/dynolog_npu/cli/src/commands/dcgm.rs @@ -0,0 +1,49 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +use std::net::TcpStream; +use rustls::{ClientConnection, StreamOwned}; + +use anyhow::Result; + +#[path = "utils.rs"] +mod utils; + +// This module contains the handling logic for dcgm + +/// Pause dcgm module profiling +pub fn run_dcgm_pause( + mut client: StreamOwned, + duration_s: i32, +) -> Result<()> { + let request_json = format!( + r#" +{{ + "fn": "dcgmProfPause", + "duration_s": {} +}}"#, + duration_s + ); + + utils::send_msg(&mut client, &request_json).expect("Error sending message to service"); + + let resp_str = utils::get_resp(&mut client).expect("Unable to decode output bytes"); + + println!("response = {}", resp_str); + + Ok(()) +} + +/// Resume dcgm module profiling +pub fn run_dcgm_resume(mut client: StreamOwned) -> Result<()> { + utils::send_msg(&mut client, r#"{"fn":"dcgmProfResume"}"#) + .expect("Error sending message to service"); + + let resp_str = utils::get_resp(&mut client).expect("Unable to decode output bytes"); + + println!("response = {}", resp_str); + + Ok(()) +} \ No newline at end of file diff --git a/msmonitor/dynolog_npu/cli/src/commands/gputrace.rs b/msmonitor/dynolog_npu/cli/src/commands/gputrace.rs new file mode 100644 index 0000000000000000000000000000000000000000..c27b7534e06a8ed8569a44dadaaf2654da093589 --- /dev/null +++ b/msmonitor/dynolog_npu/cli/src/commands/gputrace.rs @@ -0,0 +1,217 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +use std::net::TcpStream; +use rustls::{ClientConnection, StreamOwned}; + +use anyhow::Result; +use serde_json::Value; + +#[path = "utils.rs"] +mod utils; + +// This module contains the handling logic for dyno gputrace + +#[derive(Debug)] +pub enum GpuTraceTriggerConfig { + DurationBased { + profile_start_time: u64, + duration_ms: u64, + }, + IterationBased { + profile_start_iteration_roundup: u64, + iterations: i64, + }, +} + +impl GpuTraceTriggerConfig { + fn config(&self) -> String { + match *self { + GpuTraceTriggerConfig::DurationBased { + profile_start_time, + duration_ms, + } => format!( + "PROFILE_START_TIME={}\nACTIVITIES_DURATION_MSECS={}", + profile_start_time, duration_ms + ), + GpuTraceTriggerConfig::IterationBased { + profile_start_iteration_roundup, + iterations, + } => format!( + r#"PROFILE_START_ITERATION=0 +PROFILE_START_ITERATION_ROUNDUP={} +ACTIVITIES_ITERATIONS={}"#, + profile_start_iteration_roundup, iterations + ), + } + } +} + +#[derive(Debug)] +pub struct GpuTraceOptions { + pub record_shapes: bool, + pub profile_memory: bool, + pub with_stacks: bool, + pub with_flops: bool, + pub with_modules: bool, +} + +impl GpuTraceOptions { + fn config(&self) -> String { + format!( + r#" +PROFILE_REPORT_INPUT_SHAPES={} +PROFILE_PROFILE_MEMORY={} +PROFILE_WITH_STACK={} +PROFILE_WITH_FLOPS={} +PROFILE_WITH_MODULES={}"#, + self.record_shapes, + self.profile_memory, + self.with_stacks, + self.with_flops, + self.with_modules + ) + } +} + +#[derive(Debug)] +pub struct GpuTraceConfig { + pub log_file: String, + pub trigger_config: GpuTraceTriggerConfig, + pub trace_options: GpuTraceOptions, +} + +impl GpuTraceConfig { + fn config(&self) -> String { + format!( + "ACTIVITIES_LOG_FILE={}\n{}{}", + self.log_file, + self.trigger_config.config(), + self.trace_options.config() + ) + } +} + +/// Gputrace command triggers GPU profiling on pytorch apps +pub fn run_gputrace( + mut client: StreamOwned, + job_id: u64, + pids: &str, + process_limit: u32, + config: GpuTraceConfig, +) -> Result<()> { + let kineto_config = config.config(); + println!("Kineto config = \n{}", kineto_config); + let kineto_config = kineto_config.replace('\n', "\\n"); + + let request_json = format!( + r#" +{{ + "fn": "setKinetOnDemandRequest", + "config": "{}", + "job_id": {}, + "pids": [{}], + "process_limit": {} +}}"#, + kineto_config, job_id, pids, process_limit + ); + + utils::send_msg(&mut client, &request_json).expect("Error sending message to service"); + + let resp_str = utils::get_resp(&mut client).expect("Unable to decode output bytes"); + + println!("response = {}", resp_str); + + let resp_v: Value = serde_json::from_str(&resp_str)?; + let processes = resp_v["processesMatched"].as_array().unwrap(); + + if processes.is_empty() { + println!("No processes were matched, please check --job-id or --pids flags"); + } else { + println!("Matched {} processes", processes.len()); + println!("Trace output files will be written to:"); + + for pid in processes { + let pid = pid.as_i64().unwrap(); + println!( + " {}", + config.log_file.replace(".json", &format!("_{}.json", pid)) + ); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::*; + + #[test] + fn test_gputrace_trigger_config() { + let trigger_config = GpuTraceTriggerConfig::DurationBased { + profile_start_time: 1000, + duration_ms: 42, + }; + assert_eq!( + trigger_config.config(), + r#"PROFILE_START_TIME=1000 +ACTIVITIES_DURATION_MSECS=42"# + ); + + let trigger_config = GpuTraceTriggerConfig::IterationBased { + profile_start_iteration_roundup: 1000, + iterations: 42, + }; + assert_eq!( + trigger_config.config(), + r#"PROFILE_START_ITERATION=0 +PROFILE_START_ITERATION_ROUNDUP=1000 +ACTIVITIES_ITERATIONS=42"# + ); + } + + #[test] + fn test_gputrace_config() { + let mut test_trace_options = GpuTraceOptions { + record_shapes: true, + profile_memory: false, + with_stacks: true, + with_flops: false, + with_modules: true, + }; + assert_eq!( + test_trace_options.config(), + r#" +PROFILE_REPORT_INPUT_SHAPES=true +PROFILE_PROFILE_MEMORY=false +PROFILE_WITH_STACK=true +PROFILE_WITH_FLOPS=false +PROFILE_WITH_MODULES=true"# + ); + + test_trace_options.profile_memory = true; + + let test_trace_config = GpuTraceConfig { + log_file: String::from("/tmp/test_trace.json"), + trigger_config: GpuTraceTriggerConfig::DurationBased { + profile_start_time: 1000, + duration_ms: 42, + }, + trace_options: test_trace_options, + }; + assert_eq!( + test_trace_config.config(), + r#"ACTIVITIES_LOG_FILE=/tmp/test_trace.json +PROFILE_START_TIME=1000 +ACTIVITIES_DURATION_MSECS=42 +PROFILE_REPORT_INPUT_SHAPES=true +PROFILE_PROFILE_MEMORY=true +PROFILE_WITH_STACK=true +PROFILE_WITH_FLOPS=false +PROFILE_WITH_MODULES=true"# + ); + } +} \ No newline at end of file diff --git a/dynolog_npu/dynolog_npu/cli/src/commands/mod.rs b/msmonitor/dynolog_npu/cli/src/commands/mod.rs similarity index 100% rename from dynolog_npu/dynolog_npu/cli/src/commands/mod.rs rename to msmonitor/dynolog_npu/cli/src/commands/mod.rs diff --git a/dynolog_npu/dynolog_npu/cli/src/commands/npumonitor.rs b/msmonitor/dynolog_npu/cli/src/commands/npumonitor.rs similarity index 79% rename from dynolog_npu/dynolog_npu/cli/src/commands/npumonitor.rs rename to msmonitor/dynolog_npu/cli/src/commands/npumonitor.rs index 1edfaea5939f5cee5df8618720d1bfa16d0071b5..f8f73c5b959af37973552286426d6a20edea650f 100644 --- a/dynolog_npu/dynolog_npu/cli/src/commands/npumonitor.rs +++ b/msmonitor/dynolog_npu/cli/src/commands/npumonitor.rs @@ -1,3 +1,4 @@ +use rustls::{ClientConnection, StreamOwned}; use std::net::TcpStream; use anyhow::Result; @@ -30,7 +31,7 @@ MSPTI_ACTIVITY_KIND={}"#, } pub fn run_npumonitor( - client: TcpStream, + mut client: StreamOwned, config: NpuMonitorConfig, ) -> Result<()> { let config_str = config.config(); @@ -49,9 +50,9 @@ pub fn run_npumonitor( config_str ); - utils::send_msg(&client, &request_json).expect("Error sending message to service"); + utils::send_msg(&mut client, &request_json).expect("Error sending message to service"); - let resp_str = utils::get_resp(&client).expect("Unable to decode output bytes"); + let resp_str = utils::get_resp(&mut client).expect("Unable to decode output bytes"); println!("response = {}", resp_str); diff --git a/dynolog_npu/dynolog_npu/cli/src/commands/nputrace.rs b/msmonitor/dynolog_npu/cli/src/commands/nputrace.rs similarity index 82% rename from dynolog_npu/dynolog_npu/cli/src/commands/nputrace.rs rename to msmonitor/dynolog_npu/cli/src/commands/nputrace.rs index f70923bca4cc5ce29a8855a464c411b63a930ef0..f7b14f9b11ff4ee219c75de90c581cf6135590cf 100644 --- a/dynolog_npu/dynolog_npu/cli/src/commands/nputrace.rs +++ b/msmonitor/dynolog_npu/cli/src/commands/nputrace.rs @@ -1,4 +1,5 @@ use std::net::TcpStream; +use rustls::{ClientConnection, StreamOwned}; use anyhow::Result; use serde_json::Value; @@ -59,6 +60,11 @@ pub struct NpuTraceOptions { pub gc_detect_threshold: Option, pub data_simplification: String, pub export_type: String, + pub host_sys: String, + pub sys_io: bool, + pub sys_interconnection: bool, + pub mstx_domain_include: Option, + pub mstx_domain_exclude: Option, } impl NpuTraceOptions { @@ -79,7 +85,12 @@ PROFILE_OP_ATTR={} PROFILE_MSPROF_TX={} PROFILE_GC_DETECT_THRESHOLD={} PROFILE_DATA_SIMPLIFICATION={} -PROFILE_EXPORT_TYPE={}"#, +PROFILE_EXPORT_TYPE={} +PROFILE_HOST_SYS={} +PROFILE_SYS_IO={} +PROFILE_SYS_INTERCONNECTION={} +PROFILE_MSTX_DOMAIN_INCLUDE={} +PROFILE_MSTX_DOMAIN_EXCLUDE={}"#, self.record_shapes, self.profile_memory, self.with_stack, @@ -94,7 +105,12 @@ PROFILE_EXPORT_TYPE={}"#, self.msprof_tx, self.gc_detect_threshold.map_or("None".to_string(), |v| v.to_string()), self.data_simplification, - self.export_type + self.export_type, + self.host_sys, + self.sys_io, + self.sys_interconnection, + self.mstx_domain_include.clone().map_or("None".to_string(), |v| v.to_string()), + self.mstx_domain_exclude.clone().map_or("None".to_string(), |v| v.to_string()) ) } } @@ -118,7 +134,7 @@ impl NpuTraceConfig { } pub fn run_nputrace( - client: TcpStream, + mut client: StreamOwned, job_id: u64, pids: &str, process_limit: u32, @@ -140,9 +156,9 @@ pub fn run_nputrace( config_str, job_id, pids, process_limit ); - utils::send_msg(&client, &request_json).expect("Error sending message to service"); + utils::send_msg(&mut client, &request_json).expect("Error sending message to service"); - let resp_str = utils::get_resp(&client).expect("Unable to decode output bytes"); + let resp_str = utils::get_resp(&mut client).expect("Unable to decode output bytes"); println!("response = {}", resp_str); @@ -220,6 +236,11 @@ ACTIVITIES_ITERATIONS=1000"# gc_detect_threshold: 0.1, data_simplification: "true", export_type: "Text".to_string(), + host_sys: "cpu".to_string(), + sys_io: true, + sys_interconnection: true, + mstx_domain_include: "domain1".to_string(), + mstx_domain_exclude: "domain2".to_string(), }, }; assert_eq!( @@ -241,7 +262,12 @@ PROFILE_OP_ATTR=true PROFILE_MSPROF_TX=true PROFILE_GC_DETECT_THRESHOLD=0.1 PROFILE_DATA_SIMPLIFICATION=true -PROFILE_EXPORT_TYPE=Text"# +PROFILE_EXPORT_TYPE=Text +PROFILE_HOST_SYS=cpu +PROFILE_SYS_IO=true +PROFILE_SYS_INTERCONNECTION=true +PROFILE_MSTX_DOMAIN_INCLUDE=domain1 +PROFILE_MSTX_DOMAIN_EXCLUDE=domain2"# ); } } diff --git a/msmonitor/dynolog_npu/cli/src/commands/status.rs b/msmonitor/dynolog_npu/cli/src/commands/status.rs new file mode 100644 index 0000000000000000000000000000000000000000..46a56b6c64582c1b710d7cf0d8beba0c87728525 --- /dev/null +++ b/msmonitor/dynolog_npu/cli/src/commands/status.rs @@ -0,0 +1,25 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +use rustls::{ClientConnection, StreamOwned}; +use std::net::TcpStream; + +use anyhow::Result; + +#[path = "utils.rs"] +mod utils; + +// This module contains the handling logic for dyno status + +/// Get system info +pub fn run_status(mut client: StreamOwned) -> Result<()> { + utils::send_msg(&mut client, r#"{"fn":"getStatus"}"#).expect("Error sending message to service"); + + let resp_str = utils::get_resp(&mut client).expect("Unable to decode output bytes"); + + println!("response = {}", resp_str); + + Ok(()) +} \ No newline at end of file diff --git a/msmonitor/dynolog_npu/cli/src/commands/utils.rs b/msmonitor/dynolog_npu/cli/src/commands/utils.rs new file mode 100644 index 0000000000000000000000000000000000000000..ab78ec1a8ab35f75076715766a02ffb39a7682d9 --- /dev/null +++ b/msmonitor/dynolog_npu/cli/src/commands/utils.rs @@ -0,0 +1,33 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +use std::io::{Read, Write}; + +use anyhow::Result; + +pub fn send_msg(client: &mut T, msg: &str) -> Result<()> { + let msg_len: [u8; 4] = i32::try_from(msg.len()).unwrap().to_ne_bytes(); + + client.write_all(&msg_len)?; + client.write_all(msg.as_bytes()).map_err(|err| err.into()) +} + +pub fn get_resp(client: &mut T) -> Result { + // Response is prefixed with length + let mut resp_len: [u8; 4] = [0; 4]; + client.read_exact(&mut resp_len)?; + + let resp_len = i32::from_ne_bytes(resp_len); + let resp_len = usize::try_from(resp_len).unwrap(); + + println!("response length = {}", resp_len); + + let mut resp_str = Vec::::new(); + resp_str.resize(resp_len, 0); + + client.read_exact(resp_str.as_mut_slice())?; + + String::from_utf8(resp_str).map_err(|err| err.into()) +} \ No newline at end of file diff --git a/msmonitor/dynolog_npu/cli/src/commands/version.rs b/msmonitor/dynolog_npu/cli/src/commands/version.rs new file mode 100644 index 0000000000000000000000000000000000000000..5a29a85aaad3a7affe508e4c400de1b4e16beee0 --- /dev/null +++ b/msmonitor/dynolog_npu/cli/src/commands/version.rs @@ -0,0 +1,24 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +use rustls::{ClientConnection, StreamOwned}; +use std::net::TcpStream; +use anyhow::Result; + +#[path = "utils.rs"] +mod utils; + +// This module contains the handling logic for querying dyno version + +/// Get version info +pub fn run_version(mut client: StreamOwned) -> Result<()> { + utils::send_msg(&mut client, r#"{"fn":"getVersion"}"#).expect("Error sending message to service"); + + let resp_str = utils::get_resp(&mut client).expect("Unable to decode output bytes"); + + println!("response = {}", resp_str); + + Ok(()) +} \ No newline at end of file diff --git a/dynolog_npu/dynolog_npu/cli/src/main.rs b/msmonitor/dynolog_npu/cli/src/main.rs similarity index 70% rename from dynolog_npu/dynolog_npu/cli/src/main.rs rename to msmonitor/dynolog_npu/cli/src/main.rs index 9fdea3d1254467081356b2e0daeb8ed3ca05a16d..a71150a5af5c6db09e15faeb73e1ec6dc8ab4b82 100644 --- a/dynolog_npu/dynolog_npu/cli/src/main.rs +++ b/msmonitor/dynolog_npu/cli/src/main.rs @@ -2,9 +2,14 @@ // // This source code is licensed under the MIT license found in the // LICENSE file in the root directory of this source tree. - +use std::fs::File; +use std::io::BufReader; +use rustls::{Certificate, RootCertStore, PrivateKey, ClientConnection, StreamOwned}; +use std::sync::Arc; use std::net::TcpStream; use std::net::ToSocketAddrs; +use std::path::PathBuf; +use std::io; use anyhow::Result; use clap::Parser; @@ -44,6 +49,8 @@ struct Opts { hostname: String, #[clap(long, default_value_t = DYNO_PORT)] port: u16, + #[clap(long, required = true)] + certs_dir: String, #[clap(subcommand)] cmd: Command, } @@ -64,6 +71,27 @@ fn parse_mspti_activity_kinds(src: &str) -> Result{ Ok(src.to_string()) } +const ALLOWED_HOST_SYSTEM_VALUES: &[&str] = &["cpu", "mem", "disk", "network", "osrt"]; + +fn parse_host_sys(src: &str) -> Result{ + if src == "None" { + return Ok(src.to_string()); + } + + let allowed_host_sys_values: HashSet<&str> = ALLOWED_HOST_SYSTEM_VALUES.iter().cloned().collect(); + + let host_systems: Vec<&str> = src.split(',').map(|s| s.trim()).collect(); + + for host_system in &host_systems { + if !allowed_host_sys_values.contains(host_system) { + return Err(format!("Invalid NPU Trace host system: {}, Possible values: {:?}.]", host_system, + allowed_host_sys_values)); + } + } + let result = host_systems.join(","); + Ok(result) +} + #[derive(Debug, Parser)] enum Command { /// Check the status of a dynolog process @@ -184,6 +212,21 @@ enum Command { /// Types of data exported by the profiler. #[clap(long, value_parser = ["Text", "Db"], default_value = "Text")] export_type: String, + /// Obtain the system data on the host side. + #[clap(long, value_parser = parse_host_sys, default_value = "None")] + host_sys: String, + /// Whether to enable sys io. + #[clap(long, action)] + sys_io: bool, + /// Whether to enable sys interconnection. + #[clap(long, action)] + sys_interconnection: bool, + /// The domain that needs to be enabled in mstx mode. + #[clap(long)] + mstx_domain_include: Option, + /// Domains that do not need to be enabled in mstx mode. + #[clap(long)] + mstx_domain_exclude: Option, }, /// Ascend MSPTI Monitor NpuMonitor { @@ -210,29 +253,98 @@ enum Command { DcgmResume, } -/// Create a socket connection to dynolog -fn create_dyno_client(host: &str, port: u16) -> Result { +struct ClientConfigPath { + cert_path: PathBuf, + key_path: PathBuf, + ca_cert_path: PathBuf, +} + +fn create_dyno_client( + host: &str, + port: u16, + config: &ClientConfigPath +) -> Result> { let addr = (host, port) .to_socket_addrs()? .next() - .expect("Failed to connect to the server"); + .ok_or_else(|| io::Error::new( + io::ErrorKind::NotFound, + "Could not resolve the host address" + ))?; + + let stream = TcpStream::connect(addr)?; + + println!("Loading CA cert from: {}", config.ca_cert_path.display()); + let mut root_store = RootCertStore::empty(); + let ca_file = File::open(&config.ca_cert_path)?; + let mut ca_reader = BufReader::new(ca_file); + let ca_certs = rustls_pemfile::certs(&mut ca_reader)?; + for ca_cert in ca_certs { + root_store.add(&Certificate(ca_cert))?; + } + + println!("Loading client cert from: {}", config.cert_path.display()); + let cert_file = File::open(&config.cert_path)?; + let mut cert_reader = BufReader::new(cert_file); + let certs = rustls_pemfile::certs(&mut cert_reader)? + .into_iter() + .map(Certificate) + .collect(); - TcpStream::connect(addr).map_err(|err| err.into()) + println!("Loading client key from: {}", config.key_path.display()); + let key_file = File::open(&config.key_path)?; + let mut key_reader = BufReader::new(key_file); + let keys = rustls_pemfile::pkcs8_private_keys(&mut key_reader)?; + if keys.is_empty() { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "No private key found in the key file" + ).into()); + } + let key = PrivateKey(keys[0].clone()); + + let config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_root_certificates(root_store) + .with_client_auth_cert(certs, key)?; + + let server_name = rustls::ServerName::try_from(host) + .map_err(|e| io::Error::new( + io::ErrorKind::InvalidInput, + format!("Invalid hostname: {}", e) + ))?; + + let conn = rustls::ClientConnection::new( + Arc::new(config), + server_name + )?; + + // 返回 TLS stream + Ok(StreamOwned::new(conn, stream)) } fn main() -> Result<()> { let Opts { hostname, port, + certs_dir, cmd, } = Opts::parse(); - let dyno_client = - create_dyno_client(&hostname, port).expect("Couldn't connect to the server..."); + let certs_dir = PathBuf::from(&certs_dir); + + let config = ClientConfigPath { + cert_path: certs_dir.join("client.crt"), + key_path: certs_dir.join("client.key"), + ca_cert_path: certs_dir.join("ca.crt"), + }; + + let client = create_dyno_client(&hostname, port, &config) + .expect("Couldn't connect to the server..."); match cmd { - Command::Status => status::run_status(dyno_client), - Command::Version => version::run_version(dyno_client), + Command::Status => status::run_status(client), + Command::Version => version::run_version(client), Command::Gputrace { job_id, pids, @@ -271,7 +383,7 @@ fn main() -> Result<()> { trigger_config, trace_options, }; - gputrace::run_gputrace(dyno_client, job_id, &pids, process_limit, trace_config) + gputrace::run_gputrace(client, job_id, &pids, process_limit, trace_config) } Command::Nputrace { job_id, @@ -297,6 +409,11 @@ fn main() -> Result<()> { gc_detect_threshold, data_simplification, export_type, + host_sys, + sys_io, + sys_interconnection, + mstx_domain_include, + mstx_domain_exclude, } => { let trigger_config = if iterations > 0 { NpuTraceTriggerConfig::IterationBased { @@ -326,13 +443,18 @@ fn main() -> Result<()> { gc_detect_threshold, data_simplification, export_type, + host_sys, + sys_io, + sys_interconnection, + mstx_domain_include, + mstx_domain_exclude, }; let trace_config = NpuTraceConfig { log_file, trigger_config, trace_options, }; - nputrace::run_nputrace(dyno_client, job_id, &pids, process_limit, trace_config) + nputrace::run_nputrace(client, job_id, &pids, process_limit, trace_config) } Command::NpuMonitor { npu_monitor_start, @@ -346,10 +468,10 @@ fn main() -> Result<()> { report_interval_s, mspti_activity_kind }; - npumonitor::run_npumonitor(dyno_client, npu_mon_config) + npumonitor::run_npumonitor(client, npu_mon_config) } - Command::DcgmPause { duration_s } => dcgm::run_dcgm_pause(dyno_client, duration_s), - Command::DcgmResume => dcgm::run_dcgm_resume(dyno_client), + Command::DcgmPause { duration_s } => dcgm::run_dcgm_pause(client, duration_s), + Command::DcgmResume => dcgm::run_dcgm_resume(client), // ... add new commands here } } \ No newline at end of file diff --git a/msmonitor/dynolog_npu/dynolog/src/CMakeLists.txt b/msmonitor/dynolog_npu/dynolog/src/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..dfa337ec532df3eeca520e2754b9deb1fa7dea88 --- /dev/null +++ b/msmonitor/dynolog_npu/dynolog/src/CMakeLists.txt @@ -0,0 +1,71 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +set(CMAKE_SKIP_RPATH TRUE) + +cmake_minimum_required(VERSION 3.16) +add_definitions(-DDYNOLOG_VERSION=${DYNOLOG_VERSION} -DDYNOLOG_GIT_REV=${DYNOLOG_GIT_REV}) + +message("Use Prometheus = ${USE_PROMETHEUS}") +message("Use ODS Graph API = ${USE_ODS_GRAPH_API}") + +# our build script will first create a src/ dir where all source code will exist +file (GLOB dynolog_src "*.h" "*.cpp") + +# Remove main from library, only needed for exec. +list(REMOVE_ITEM dynolog_src "${CMAKE_CURRENT_SOURCE_DIR}/Main.cpp") +add_library(dynolog_lib ${dynolog_src}) + +if(USE_ODS_GRAPH_API) + target_compile_options(dynolog_lib PUBLIC "-DUSE_GRAPH_ENDPOINT") +endif() + +if(USE_PROMETHEUS) + find_package(prometheus-cpp CONFIG REQUIRED) + add_definitions(-DUSE_PROMETHEUS) + target_link_libraries(dynolog_lib PRIVATE prometheus-cpp::pull) +endif() + +target_compile_options(dynolog_lib PRIVATE + -fPIC + -fstack-protector-all + -ftrapv +) + +target_link_options(dynolog_lib PRIVATE + -Wl,-z,relro,-z,now,-z,noexecstack + -s +) + +target_link_libraries(dynolog_lib PUBLIC Monitor) +target_link_libraries(dynolog_lib PUBLIC BuiltinMetrics) + +add_subdirectory(rpc) + +add_subdirectory(ipcfabric) +target_link_libraries(dynolog_lib PUBLIC dynolog_ipcfabric_lib) + +# depends on ipcfabric +add_subdirectory(tracing) +target_link_libraries(dynolog_lib PUBLIC dynolog_ipcmonitor_lib) + +add_subdirectory(gpumon) +target_link_libraries(dynolog_lib PUBLIC dynolog_dcgm_lib "-ldl") + +add_subdirectory(rdmamon) +target_link_libraries(dynolog_lib PUBLIC dynolog_rdmamon_lib) + +add_subdirectory(metric_frame) + +add_executable(dynolog Main.cpp) +target_link_libraries(dynolog PRIVATE dynolog_lib dynolog_rpc_lib) + +target_compile_options(dynolog PRIVATE + -fPIC + -fstack-protector-all + -ftrapv +) + +target_link_options(dynolog PRIVATE + -Wl,-z,relro,-z,now,-z,noexecstack + -s +) \ No newline at end of file diff --git a/dynolog_npu/dynolog_npu/dynolog/src/Main.cpp b/msmonitor/dynolog_npu/dynolog/src/Main.cpp similarity index 93% rename from dynolog_npu/dynolog_npu/dynolog/src/Main.cpp rename to msmonitor/dynolog_npu/dynolog/src/Main.cpp index 8e5177768327e37173d4e7661e334a9400bd6172..758d9db3ed9a2a153d94ee9f167811cc0d9a69f8 100644 --- a/dynolog_npu/dynolog_npu/dynolog/src/Main.cpp +++ b/msmonitor/dynolog_npu/dynolog/src/Main.cpp @@ -166,13 +166,16 @@ int main(int argc, char** argv) { std::shared_ptr dcgm; std::unique_ptr ipcmon; - std::unique_ptr ipcmon_thread, gpumon_thread, pm_thread; + std::unique_ptr ipcmon_thread, data_ipcmon_thread, gpumon_thread, pm_thread; if (FLAGS_enable_ipc_monitor) { LOG(INFO) << "Starting IPC Monitor"; ipcmon = std::make_unique(); + ipcmon->setLogger(std::move(getLogger())); ipcmon_thread = std::make_unique([&ipcmon]() { ipcmon->loop(); }); + data_ipcmon_thread = + std::make_unique([&ipcmon]() { ipcmon->dataLoop(); }); } if (FLAGS_enable_gpu_monitor) { @@ -192,15 +195,18 @@ int main(int argc, char** argv) { auto server = setup_server(handler); server->run(); - km_thread.join(); - if (pm_thread) { + if (km_thread.joinable()) { + km_thread.join(); + } + + if (pm_thread && pm_thread->joinable()) { pm_thread->join(); } - if (gpumon_thread) { + if (gpumon_thread && gpumon_thread->joinable()) { gpumon_thread->join(); } server->stop(); return 0; -} \ No newline at end of file +} diff --git a/msmonitor/dynolog_npu/dynolog/src/Metric.cpp b/msmonitor/dynolog_npu/dynolog/src/Metric.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f6fd4d80de13f3819abc0e519e31c0890bd8c141 --- /dev/null +++ b/msmonitor/dynolog_npu/dynolog/src/Metric.cpp @@ -0,0 +1,37 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +#include "dynolog/src/Metrics.h" + +#include +#include + +namespace dynolog { + +const std::vector getAllMetrics() { + static std::vector metrics_ = { + {.name = "kindName", + .type = MetricType::Instant, + .desc = "Report data kind name"}, + {.name = "duration", + .type = MetricType::Delta, + .desc = "Total execution time for corresponding kind"}, + {.name = "timestamp", + .type = MetricType::Instant, + .desc = "The timestamp of the reported data"}, + {.name = "deviceId", + .type = MetricType::Instant, + .desc = "The ID of the device for reporting data"}, + }; + return metrics_; +} + +// These metrics are dynamic per network drive +const std::vector getNetworkMetrics() { + static std::vector metrics_ = {}; + return metrics_; +} + +} // namespace dynolog \ No newline at end of file diff --git a/msmonitor/dynolog_npu/dynolog/src/rpc/CMakeLists.txt b/msmonitor/dynolog_npu/dynolog/src/rpc/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..a0b74f82cf9be5cec400e6477183b15a52b76cdc --- /dev/null +++ b/msmonitor/dynolog_npu/dynolog/src/rpc/CMakeLists.txt @@ -0,0 +1,20 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +find_package(OpenSSL REQUIRED) + +add_library(dynolog_rpc_lib STATIC + SimpleJsonServer.cpp SimpleJsonServer.h + ${CMAKE_CURRENT_SOURCE_DIR}/../ServiceHandler.h +) +target_include_directories(dynolog_rpc_lib + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} +) + +target_include_directories(dynolog_rpc_lib + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/.. +) +target_link_libraries(dynolog_rpc_lib PRIVATE dynolog_lib) +target_link_libraries(dynolog_rpc_lib PUBLIC gflags::gflags) +target_link_libraries(dynolog_rpc_lib PUBLIC glog::glog) +target_link_libraries(dynolog_rpc_lib PUBLIC nlohmann_json::nlohmann_json) +target_link_libraries(dynolog_rpc_lib PUBLIC fmt::fmt) +target_link_libraries(dynolog_rpc_lib PRIVATE OpenSSL::SSL OpenSSL::Crypto) \ No newline at end of file diff --git a/msmonitor/dynolog_npu/dynolog/src/rpc/SimpleJsonServer.cpp b/msmonitor/dynolog_npu/dynolog/src/rpc/SimpleJsonServer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..17a6d42895b8d81a8818defa6defd3a5f3ffd1c6 --- /dev/null +++ b/msmonitor/dynolog_npu/dynolog/src/rpc/SimpleJsonServer.cpp @@ -0,0 +1,290 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +#include "dynolog/src/rpc/SimpleJsonServer.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +DEFINE_string(certs_dir, "", "TLS crets dir"); + +constexpr int CLIENT_QUEUE_LEN = 50; + +namespace dynolog { + +SimpleJsonServerBase::SimpleJsonServerBase(int port) : port_(port) { + initSocket(); + init_openssl(); + ctx_ = create_context(); + configure_context(ctx_); +} + +SimpleJsonServerBase::~SimpleJsonServerBase() { + if (thread_) { + stop(); + } + close(sock_fd_); +} + +void SimpleJsonServerBase::initSocket() { + struct sockaddr_in6 server_addr; + + /* Create socket for listening (client requests).*/ + sock_fd_ = ::socket(AF_INET6, SOCK_STREAM, 0); + if (sock_fd_ == -1) { + std::perror("socket()"); + return; + } + + /* Set socket to reuse address in case server is restarted.*/ + int flag = 1; + int ret = + ::setsockopt(sock_fd_, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); + if (ret == -1) { + std::perror("setsockopt()"); + return; + } + + // in6addr_any allows us to bind to both IPv4 and IPv6 clients. + server_addr.sin6_addr = in6addr_any; + server_addr.sin6_family = AF_INET6; + server_addr.sin6_port = htons(port_); + + /* Bind address and socket together */ + ret = ::bind(sock_fd_, (struct sockaddr*)&server_addr, sizeof(server_addr)); + if (ret == -1) { + std::perror("bind()"); + close(sock_fd_); + return; + } + + /* Create listening queue (client requests) */ + ret = ::listen(sock_fd_, CLIENT_QUEUE_LEN); + if (ret == -1) { + std::perror("listen()"); + close(sock_fd_); + return; + } + + /* Get port if assigned 0 */ + if (port_ == 0) { + socklen_t len_out = sizeof(server_addr); + ret = ::getsockname(sock_fd_, (struct sockaddr*)&server_addr, &len_out); + if (ret < 0 || len_out != sizeof(server_addr)) { + std::perror("getsockname()"); + } else { + port_ = ntohs(server_addr.sin6_port); + LOG(INFO) << "System assigned port = " << ntohs(server_addr.sin6_port); + } + } + + LOG(INFO) << "Listening to connections on port " << port_; + initSuccess_ = true; +} + +/* A simple wrapper to accept connections and read data + * + * Messages are prefixed using the length so we know how long a message + * to actually read. + * : int32_t len + * : char json[] + */ +class ClientSocketWrapper { + public: + ~ClientSocketWrapper() { + if (ssl_) { + SSL_shutdown(ssl_); + SSL_free(ssl_); + } + if (client_sock_fd_ != -1) { + ::close(client_sock_fd_); + } + } + + bool accept(int server_socket, SSL_CTX* ctx) { + struct sockaddr_in6 client_addr; + socklen_t client_addr_len = sizeof(client_addr); + std::array client_addr_str; + + client_sock_fd_ = ::accept( + server_socket, (struct sockaddr*)&client_addr, &client_addr_len); + if (client_sock_fd_ == -1) { + std::perror("accept()"); + return false; + } + + inet_ntop( + AF_INET6, + &(client_addr.sin6_addr), + client_addr_str.data(), + client_addr_str.size()); + LOG(INFO) << "Received connection from " << client_addr_str.data(); + + ssl_ = SSL_new(ctx); + SSL_set_fd(ssl_, client_sock_fd_); + if (SSL_accept(ssl_) <= 0) { + ERR_print_errors_fp(stderr); + return false; + } + LOG(INFO) << "SSL handshake success"; + return true; + } + + std::string get_message() { + int32_t msg_size = -1; + if (!read_helper((uint8_t*)&msg_size, sizeof(msg_size)) || msg_size <= 0) { + LOG(ERROR) << "Invalid message size = " << msg_size; + return ""; + } + std::string message; + message.resize(msg_size); + int recv = 0; + int ret = 1; + while (recv < msg_size && ret > 0) { + ret = read_helper((uint8_t*)&message[recv], msg_size - recv); + recv += ret > 0 ? ret : 0; + } + if (recv != msg_size) { + LOG(ERROR) << "Received partial message, expected size " << msg_size + << " found : " << recv; + LOG(ERROR) << "Message received = " << message; + return ""; + } + return message; + } + + bool send_response(const std::string& response) { + int32_t size = response.size(); + int ret = SSL_write(ssl_, (void*)&size, sizeof(size)); + if (ret <= 0) { + ERR_print_errors_fp(stderr); + return false; + } + int sent = 0; + while (sent < size && ret > 0) { + ret = SSL_write(ssl_, (void*)&response[sent], size - sent); + if (ret <= 0) { + ERR_print_errors_fp(stderr); + } else { + sent += ret; + } + } + if (sent < response.size()) { + LOG(ERROR) << "Unable to write full response"; + return false; + } + return ret > 0; + } + + private: + int read_helper(uint8_t* buf, int size) { + int ret = SSL_read(ssl_, (void*)buf, size); + if (ret <= 0) { + ERR_print_errors_fp(stderr); + } + return ret; + } + + int client_sock_fd_ = -1; + SSL* ssl_ = nullptr; +}; + +/* Accepts socket connections and processes the payloads. + * This will inturn call the Handler functions*/ +void SimpleJsonServerBase::loop() noexcept { + if (sock_fd_ == -1 || !initSuccess_) { + return; + } + + while (run_) { + processOne(); + } +} + +void SimpleJsonServerBase::processOne() noexcept { + LOG(INFO) << "Waiting for connection."; + ClientSocketWrapper client; + if (!client.accept(sock_fd_, ctx_)) { + return; + } + std::string request_str = client.get_message(); + LOG(INFO) << "RPC message received = " << request_str; + auto response_str = processOneImpl(request_str); + if (response_str.empty()) { + return; + } + if (!client.send_response(response_str)) { + LOG(ERROR) << "Failed to send response"; + } +} + +void SimpleJsonServerBase::run() { + LOG(INFO) << "Launching RPC thread"; + thread_ = std::make_unique([this]() { this->loop(); }); +} + +void SimpleJsonServerBase::init_openssl() +{ + SSL_load_error_strings(); + OpenSSL_add_ssl_algorithms(); +} + +SSL_CTX* SimpleJsonServerBase::create_context() +{ + const SSL_METHOD* method = TLS_server_method(); + SSL_CTX* ctx = SSL_CTX_new(method); + if (!ctx) { + perror("Unable to create SSL context"); + ERR_print_errors_fp(stderr); + exit(EXIT_FAILURE); + } + return ctx; +} + +void SimpleJsonServerBase::configure_context(SSL_CTX* ctx) +{ + if (FLAGS_certs_dir.empty()) { + LOG(ERROR) << "--certs-dir must be specified!"; + exit(EXIT_FAILURE); + } + + std::string certs_dir = FLAGS_certs_dir; + if (!certs_dir.empty() && certs_dir.back() != '/') + certs_dir += '/'; + + std::string server_cert = certs_dir + "server.crt"; + std::string server_key = certs_dir + "server.key"; + std::string ca_cert = certs_dir + "ca.crt"; + + LOG(INFO) << "Loading server cert: " << server_cert; + LOG(INFO) << "Loading server key: " << server_key; + LOG(INFO) << "Loading CA cert: " << ca_cert; + + // 加载服务器证书 + if (SSL_CTX_use_certificate_file(ctx, server_cert.c_str(), SSL_FILETYPE_PEM) <= 0) { + ERR_print_errors_fp(stderr); + exit(EXIT_FAILURE); + } + // 加载服务器私钥 + if (SSL_CTX_use_PrivateKey_file(ctx, server_key.c_str(), SSL_FILETYPE_PEM) <= 0 ) { + ERR_print_errors_fp(stderr); + exit(EXIT_FAILURE); + } + // 加载CA证书，实现客户端证书校验 + if (SSL_CTX_load_verify_locations(ctx, ca_cert.c_str(), NULL) <= 0) { + ERR_print_errors_fp(stderr); + exit(EXIT_FAILURE); + } + // 要求客户端必须提供证书 + SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT, NULL); +} + +} // namespace dynolog \ No newline at end of file diff --git a/msmonitor/dynolog_npu/dynolog/src/rpc/SimpleJsonServer.h b/msmonitor/dynolog_npu/dynolog/src/rpc/SimpleJsonServer.h new file mode 100644 index 0000000000000000000000000000000000000000..df5d66f75b54e88dd4c0dff01b7c28ef545cb106 --- /dev/null +++ b/msmonitor/dynolog_npu/dynolog/src/rpc/SimpleJsonServer.h @@ -0,0 +1,71 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "dynolog/src/ServiceHandler.h" + +DECLARE_string(certs_dir); + +namespace dynolog { + +// This is a simple service built using UNIX Sockets +// with remote procedure calls implemented via JSON string. + +class SimpleJsonServerBase { + public: + explicit SimpleJsonServerBase(int port); + virtual ~SimpleJsonServerBase(); + + int getPort() const { + return port_; + } + + bool initSuccessful() const { + return initSuccess_; + } + // spin up a new thread to process requets + void run(); + + void stop() { + run_ = 0; + thread_->join(); + } + + // synchronously processes a request + void processOne() noexcept; + + protected: + void initSocket(); + void init_openssl(); + SSL_CTX* create_context(); + void configure_context(SSL_CTX* ctx); + + // process requests in a loop + void loop() noexcept; + + // implement processing of request using the handler + virtual std::string processOneImpl(const std::string& request_str) { + return ""; + } + + int port_; + int sock_fd_{-1}; + bool initSuccess_{false}; + + std::atomic run_{true}; + std::unique_ptr thread_; + + SSL_CTX* ctx_{nullptr}; +}; + +} // namespace dynolog \ No newline at end of file diff --git a/msmonitor/dynolog_npu/dynolog/src/tracing/CMakeLists.txt b/msmonitor/dynolog_npu/dynolog/src/tracing/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..4afd436bcc378db13f6b925fbd319c7b381a5f2b --- /dev/null +++ b/msmonitor/dynolog_npu/dynolog/src/tracing/CMakeLists.txt @@ -0,0 +1,16 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. + +add_library (dynolog_ipcmonitor_lib IPCMonitor.cpp IPCMonitor.h + ${CMAKE_CURRENT_SOURCE_DIR}/../LibkinetoConfigManager.h +) + +target_include_directories(dynolog_ipcmonitor_lib + INTERFACE ${CMAKE_CURRENT_SOURCE_DIR} +) +target_include_directories(dynolog_ipcmonitor_lib + PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/.. +) + +target_link_libraries(dynolog_ipcmonitor_lib PUBLIC glog::glog) +target_link_libraries(dynolog_ipcmonitor_lib PUBLIC dynolog_ipcfabric_lib) +target_link_libraries(dynolog_ipcmonitor_lib PUBLIC nlohmann_json::nlohmann_json) diff --git a/msmonitor/dynolog_npu/dynolog/src/tracing/IPCMonitor.cpp b/msmonitor/dynolog_npu/dynolog/src/tracing/IPCMonitor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..811bae4e0dea1b72b6512f7d3e1819433cb1b14a --- /dev/null +++ b/msmonitor/dynolog_npu/dynolog/src/tracing/IPCMonitor.cpp @@ -0,0 +1,180 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +#include "dynolog/src/tracing/IPCMonitor.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dynolog/src/LibkinetoConfigManager.h" +#include "dynolog/src/ipcfabric/Utils.h" + +namespace dynolog { +namespace tracing { + +constexpr int kSleepUs = 10000; +constexpr int kDataMsgSleepUs = 1000; +const std::string kLibkinetoRequest = "req"; +const std::string kLibkinetoContext = "ctxt"; +const std::string kLibkinetoData = "data"; + +IPCMonitor::IPCMonitor(const std::string& ipc_fabric_name) { + ipc_manager_ = FabricManager::factory(ipc_fabric_name); + data_ipc_manager_ = FabricManager::factory(ipc_fabric_name + "_data"); + // below ensures singleton exists + LOG(INFO) << "Kineto config manager : active processes = " + << LibkinetoConfigManager::getInstance()->processCount("0"); +} + +void IPCMonitor::loop() { + while (ipc_manager_) { + if (ipc_manager_->recv()) { + std::unique_ptr msg = ipc_manager_->retrieve_msg(); + processMsg(std::move(msg)); + } + /* sleep override */ + usleep(kSleepUs); + } +} + +void IPCMonitor::dataLoop() { + while (data_ipc_manager_) { + if (data_ipc_manager_->recv()) { + std::unique_ptr msg = data_ipc_manager_->retrieve_msg(); + processDataMsg(std::move(msg)); + } + /* sleep override */ + usleep(kDataMsgSleepUs); + } +} + +void IPCMonitor::processMsg(std::unique_ptr msg) { + if (!ipc_manager_) { + LOG(ERROR) << "Fabric Manager not initialized"; + return; + } + // sizeof(msg->metadata.type) = 32, well above the size of the constant + // strings we are comparing against. memcmp is safe + if (memcmp( // NOLINT(facebook-security-vulnerable-memcmp) + msg->metadata.type, + kLibkinetoContext.data(), + kLibkinetoContext.size()) == 0) { + registerLibkinetoContext(std::move(msg)); + } else if ( + memcmp( // NOLINT(facebook-security-vulnerable-memcmp) + msg->metadata.type, + kLibkinetoRequest.data(), + kLibkinetoRequest.size()) == 0) { + getLibkinetoOnDemandRequest(std::move(msg)); + } else { + LOG(ERROR) << "TYPE UNKOWN: " << msg->metadata.type; + } +} + +void tracing::IPCMonitor::setLogger(std::unique_ptr logger) +{ + logger_ = std::move(logger); +} + +void IPCMonitor::LogData(const nlohmann::json& result) +{ + auto timestamp = result["timestamp"].get(); + logger_->logUint("timestamp", timestamp); + auto duration = result["duration"].get(); + logger_->logUint("duration", duration); + auto deviceId = result["deviceId"].get(); + logger_->logUint("deviceId", deviceId); + logger_->finalize(); +} + +void IPCMonitor::processDataMsg(std::unique_ptr msg) +{ + if (!data_ipc_manager_) { + LOG(ERROR) << "Fabric Manager not initialized"; + return; + } + if (memcmp( // NOLINT(facebook-security-vulnerable-memcmp) + msg->metadata.type, + kLibkinetoData.data(), + kLibkinetoData.size()) == 0) { + std::string message = std::string((char*)msg->buf.get(), msg->metadata.size); + try { + nlohmann::json result = nlohmann::json::parse(message); + LOG(INFO) << "Received data message : " << result; + LogData(result); + } catch (nlohmann::json::parse_error&) { + LOG(ERROR) << "Error parsing message = " << message; + return; + } + } else { + LOG(ERROR) << "TYPE UNKOWN: " << msg->metadata.type; + } +} + +void IPCMonitor::getLibkinetoOnDemandRequest( + std::unique_ptr msg) { + if (!ipc_manager_) { + LOG(ERROR) << "Fabric Manager not initialized"; + return; + } + std::string ret_config = ""; + ipcfabric::LibkinetoRequest* req = + (ipcfabric::LibkinetoRequest*)msg->buf.get(); + if (req->n == 0) { + LOG(ERROR) << "Missing pids parameter for type " << req->type; + return; + } + std::vector pids(req->pids, req->pids + req->n); + try { + ret_config = LibkinetoConfigManager::getInstance()->obtainOnDemandConfig( + std::to_string(req->jobid), pids, req->type); + VLOG(0) << "getLibkinetoOnDemandRequest() : job id " << req->jobid + << " pids = " << pids[0]; + } catch (const std::runtime_error& ex) { + LOG(ERROR) << "Kineto config manager exception : " << ex.what(); + } + std::unique_ptr ret = + ipcfabric::Message::constructMessage( + ret_config, kLibkinetoRequest); + if (!ipc_manager_->sync_send(*ret, msg->src)) { + LOG(ERROR) << "Failed to return config to libkineto: IPC sync_send fail"; + } + + return; +} + +void IPCMonitor::registerLibkinetoContext( + std::unique_ptr msg) { + if (!ipc_manager_) { + LOG(ERROR) << "Fabric Manager not initialized"; + return; + } + ipcfabric::LibkinetoContext* ctxt = + (ipcfabric::LibkinetoContext*)msg->buf.get(); + int32_t size = -1; + try { + size = LibkinetoConfigManager::getInstance()->registerLibkinetoContext( + std::to_string(ctxt->jobid), ctxt->pid, ctxt->gpu); + } catch (const std::runtime_error& ex) { + LOG(ERROR) << "Kineto config manager exception : " << ex.what(); + } + std::unique_ptr ret = + ipcfabric::Message::constructMessage( + size, kLibkinetoContext); + if (!ipc_manager_->sync_send(*ret, msg->src)) { + LOG(ERROR) << "Failed to send ctxt from dyno: IPC sync_send fail"; + } + + return; +} + +} // namespace tracing +} // namespace dynolog diff --git a/msmonitor/dynolog_npu/dynolog/src/tracing/IPCMonitor.h b/msmonitor/dynolog_npu/dynolog/src/tracing/IPCMonitor.h new file mode 100644 index 0000000000000000000000000000000000000000..1dc0cd2345fd7d7e556bc5c95361206e0fe2d7f2 --- /dev/null +++ b/msmonitor/dynolog_npu/dynolog/src/tracing/IPCMonitor.h @@ -0,0 +1,45 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. +// +// This source code is licensed under the MIT license found in the +// LICENSE file in the root directory of this source tree. + +#pragma once + +#include + +// Use glog for FabricManager.h +#define USE_GOOGLE_LOG + +#include "dynolog/src/ipcfabric/FabricManager.h" +#include "dynolog/src/Logger.h" + +namespace dynolog { +namespace tracing { + +class IPCMonitor { + public: + using FabricManager = dynolog::ipcfabric::FabricManager; + IPCMonitor(const std::string& ipc_fabric_name = "dynolog"); + virtual ~IPCMonitor() {} + + void loop(); + void dataLoop(); + + public: + virtual void processMsg(std::unique_ptr msg); + virtual void processDataMsg(std::unique_ptr msg); + void getLibkinetoOnDemandRequest(std::unique_ptr msg); + void registerLibkinetoContext(std::unique_ptr msg); + void setLogger(std::unique_ptr logger); + void LogData(const nlohmann::json& result); + + std::unique_ptr ipc_manager_; + std::unique_ptr data_ipc_manager_; + std::unique_ptr logger_; + + // friend class test_case_name##_##test_name##_Test + friend class IPCMonitorTest_LibkinetoRegisterAndOndemandTest_Test; +}; + +} // namespace tracing +} // namespace dynolog diff --git a/msmonitor/plugin/CMakeLists.txt b/msmonitor/plugin/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..f50ea06973c7539a937a0b60bb67dcf3a857396c --- /dev/null +++ b/msmonitor/plugin/CMakeLists.txt @@ -0,0 +1,67 @@ +cmake_minimum_required(VERSION 3.16) +project(IPCMonitor) + +set(CMAKE_SKIP_RPATH TRUE) + +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +find_package(pybind11 REQUIRED) +find_package(Python REQUIRED COMPONENTS Interpreter Development) + +include_directories( + ${CMAKE_CURRENT_SOURCE_DIR}/ipc_monitor + ${CMAKE_CURRENT_SOURCE_DIR}/ipc_monitor/metric + ${CMAKE_CURRENT_SOURCE_DIR}/ipc_monitor/mspti_monitor + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/securec/include + ${DYNOLOG_PATH}/third_party/glog/src + ${DYNOLOG_PATH}/build/third_party/glog + ${DYNOLOG_PATH}/third_party/json/single_include +) + +file(GLOB_RECURSE IPC_SOURCES + ${CMAKE_CURRENT_SOURCE_DIR}/ipc_monitor/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ipc_monitor/metric/*.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/ipc_monitor/mspti_monitor/*.cpp +) + +file(GLOB_RECURSE SECUREC_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/third_party/securec/src/*.c) + +set(SOURCES + bindings.cpp + ${IPC_SOURCES} + ${SECUREC_SOURCES} +) + +add_library(IPCMonitor MODULE ${SOURCES}) + +set_target_properties(IPCMonitor + PROPERTIES + OUTPUT_NAME IPCMonitor + PREFIX "" +) + +target_link_libraries(IPCMonitor PRIVATE + pybind11::module + pthread + ${CMAKE_CURRENT_SOURCE_DIR}/stub/libmspti.so +) + +target_link_libraries(IPCMonitor PRIVATE ${DYNOLOG_PATH}/build/third_party/glog/libglog.a) + +target_compile_options(IPCMonitor PRIVATE + -fPIC + -fstack-protector-all + -ftrapv + $<$>:-O2> +) + +target_link_options(IPCMonitor PRIVATE + -Wl,-z,relro,-z,now,-z,noexecstack + -s +) + +install(TARGETS IPCMonitor + DESTINATION ${CMAKE_INSTALL_PREFIX}/python-package +) diff --git a/dynolog_npu/plugin/README.md b/msmonitor/plugin/README.md similarity index 50% rename from dynolog_npu/plugin/README.md rename to msmonitor/plugin/README.md index 0cd51633bb15504416da85a49845a0ffb53d4452..43784413c9582f15f8014e863415d9ec2f422ed6 100644 --- a/dynolog_npu/plugin/README.md +++ b/msmonitor/plugin/README.md @@ -1,42 +1,49 @@ - - -# Plugins for Dynolog NPU -## 模块说明 -### IPCMonitor -提供IPC(Inter-Process Communication)通信接口，用于实现 -1. IPC控制通道: profiler backend向dynolog daemon获取profiler配置 - - -__PyDynamicMonitorProxy__: -* `init_dyno` 向dynolog daemon发送注册请求 - * input: npuId(int) - * return： None -* `poll_dyno` 向dynolog daemon获取Profiler控制参数 - * input: None - * return: str, 返回控制参数 - -## 安装方式 -### 1. 通过shell脚本一键安装 -``` -chmod +x build.sh -./build.sh -``` -### 2. 手动安装 -* 安装依赖 -``` -pip install wheel -pip install pybind11 -``` -* 编译whl包 -``` -python3 setup.py bdist_wheel -``` -以上命令执行完成后在plugn/dist目录下生成dynolog_npu插件whl安装包dynolog-npu-plugin-{version}.whl -* 安装 -``` -pip install dist/{dynolog-npu-plugin-{version}.wheel} -``` -* 卸载 -``` -pip uninstall dynolog-npu-plugin -``` \ No newline at end of file + + +# Plugins for msMonitor +## 模块说明 +### IPCMonitor +提供IPC(Inter-Process Communication)通信接口，用于实现 +1. IPC控制通道: profiler backend向dynolog daemon获取profiler配置 +2. IPC数据通道: mspti monitor向dynolog daemon发送性能数据 + +__PyDynamicMonitorProxy__: +* `init_dyno` 向dynolog daemon发送注册请求 + * input: npuId(int) + * return：None +* `poll_dyno` 向dynolog daemon获取Profiler控制参数 + * input: None + * return: str, 返回控制参数 +* `enable_dyno_npu_monitor` 开启mspti监控 + * input: cfg_map(Dict[str,str]) 配置 + * return: None + +## 安装方式 +### 1. 通过shell脚本一键安装 +``` +chmod +x build.sh +./build.sh +``` +### 2. 手动安装 +* 安装依赖 +``` +pip install wheel +pip install pybind11 +``` +* 编译whl包 +``` +bash ./stub/build_stub.sh +python3 setup.py bdist_wheel +``` +以上命令执行完成后在dist目录下生成msMonitor插件whl安装包msmonitor-plugin-{version}.whl +* 安装 +``` +pip install dist/{msmonitor-plugin-{version}.whl} +``` +* 卸载 +``` +pip uninstall msmonitor-plugin +``` + +## 日志 +* 用户可以通过配置MSMONITOR_LOG_PATH环境变量，指定日志文件路径，默认路径为当前目录下的msmonitor_log diff --git a/msmonitor/plugin/bindings.cpp b/msmonitor/plugin/bindings.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b08f7e3e3df0c9fb0d2905cd7463480bf1b17b7d --- /dev/null +++ b/msmonitor/plugin/bindings.cpp @@ -0,0 +1,30 @@ +#include +#include +#include "ipc_monitor/PyDynamicMonitorProxy.h" + +namespace py = pybind11; + +void init_IPCMonitor(PyObject *module) { + py::class_(module, "PyDynamicMonitorProxy") + .def(py::init<>()) + .def("init_dyno", &dynolog_npu::ipc_monitor::PyDynamicMonitorProxy::InitDyno, py::arg("npuId")) + .def("poll_dyno", &dynolog_npu::ipc_monitor::PyDynamicMonitorProxy::PollDyno) + .def("enable_dyno_npu_monitor", &dynolog_npu::ipc_monitor::PyDynamicMonitorProxy::EnableMsptiMonitor, py::arg("cfg_map")) + .def("finalize_dyno", &dynolog_npu::ipc_monitor::PyDynamicMonitorProxy::FinalizeDyno); +} + +static PyMethodDef g_moduleMethods[] = {}; + +static struct PyModuleDef ipcMonitor_module = { + PyModuleDef_HEAD_INIT, + "IPCMonitor", + nullptr, + -1, + g_moduleMethods +}; + +PyMODINIT_FUNC PyInit_IPCMonitor(void) { + PyObject* m = PyModule_Create(&ipcMonitor_module); + init_IPCMonitor(m); + return m; +} \ No newline at end of file diff --git a/dynolog_npu/plugin/build.sh b/msmonitor/plugin/build.sh old mode 100755 new mode 100644 similarity index 84% rename from dynolog_npu/plugin/build.sh rename to msmonitor/plugin/build.sh index ce20d9d2be546afbc63e3aace524f74858eff6ff..ec20536715a9b2bd1fd8ab7a694ca9eac26f3101 --- a/dynolog_npu/plugin/build.sh +++ b/msmonitor/plugin/build.sh @@ -3,7 +3,10 @@ # install pybind11 pip install pybind11 -# build dynolog_npu_plugin wheel +# build stub +sh ./stub/build_stub.sh + +# build msmonitor_plugin wheel python3 setup.py bdist_wheel # find .whl files in dist diff --git a/msmonitor/plugin/ipc_monitor/DynoLogNpuMonitor.cpp b/msmonitor/plugin/ipc_monitor/DynoLogNpuMonitor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..001e23b9e0ab8f173fd07763f896a6d8230ed5ea --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/DynoLogNpuMonitor.cpp @@ -0,0 +1,114 @@ +#include "DynoLogNpuMonitor.h" +#include +#include +#include +#include "utils.h" + +namespace dynolog_npu { +namespace ipc_monitor { +DynoLogNpuMonitor::DynoLogNpuMonitor() +{ + // init glog + if (!google::IsGoogleLoggingInitialized()) { + std::string logPath; + if (CreateMsmonitorLogPath(logPath)) { + fprintf(stderr, "[INFO] [%d] Msmonitor log will record to %s\n", GetProcessId(), logPath.c_str()); + logPath = logPath + "/msmonitor_"; + google::InitGoogleLogging("MsMonitor"); + google::SetLogDestination(google::GLOG_INFO, logPath.c_str()); + google::SetLogFilenameExtension(".log"); + } else { + fprintf(stderr, "Failed to create log path, log will not record\n"); + } + } +} + +bool DynoLogNpuMonitor::Init() +{ + if (isInitialized_) { + LOG(WARNING) << "DynoLog npu monitor already initialized"; + return true; + } + if (!ipcClient_.Init()) { + LOG(ERROR) << "DynoLog npu monitor ipcClient init failed"; + return false; + } + bool res = ipcClient_.RegisterInstance(npuId_); + if (res) { + isInitialized_ = true; + LOG(INFO) << "DynoLog npu monitor initialized successfully"; + } + return res; +} + +ErrCode DynoLogNpuMonitor::DealMonitorReq(MsptiMonitorCfg& cmd) +{ + if (cmd.monitorStop) { + if (msptiMonitor_.IsStarted()) { + LOG(INFO) << "Stop mspti monitor thread successfully"; + msptiMonitor_.Stop(); + } + return ErrCode::SUC; + } + + if (cmd.reportIntervals <= 0) { + cmd.reportIntervals = DEFAULT_FLUSH_INTERVAL; + LOG(WARNING) << "Invalid report interval, set to 60"; + } + if (cmd.reportIntervals != 0) { + msptiMonitor_.SetFlushInterval(cmd.reportIntervals); + } + + if (cmd.monitorStart && !msptiMonitor_.IsStarted()) { + LOG(INFO) << "Start mspti monitor thread successfully"; + msptiMonitor_.Start(); + } + + if (msptiMonitor_.IsStarted() && !cmd.enableActivities.empty()) { + auto curActivities = msptiMonitor_.GetEnabledActivities(); + std::vector enableKinds, disableKinds; + std::set_difference(cmd.enableActivities.begin(), cmd.enableActivities.end(), curActivities.begin(), curActivities.end(), + std::back_inserter(enableKinds)); + std::set_difference(curActivities.begin(), curActivities.end(), cmd.enableActivities.begin(), cmd.enableActivities.end(), + std::back_inserter(disableKinds)); + for (auto activity : enableKinds) { + msptiMonitor_.EnableActivity(activity); + } + for (auto activity : disableKinds) { + msptiMonitor_.DisableActivity(activity); + } + } + return ErrCode::SUC; +} + +std::string DynoLogNpuMonitor::Poll() +{ + std::string res = ipcClient_.IpcClientNpuConfig(); + if (res.size() == 4) { // res为4，表示dynolog注册进程成功 + LOG(INFO) << "Regist to dynolog daemon successfully"; + return ""; + } + if (res.empty()) { + return ""; + } + LOG(INFO) << "Received NPU configuration successfully"; + return res; +} + +void DynoLogNpuMonitor::EnableMsptiMonitor(std::unordered_map& cfg_map) +{ + auto cmd = InputParser::GetInstance()->DynoLogGetOpts(cfg_map); + if (cmd.isMonitor) { + auto ans = DealMonitorReq(cmd); + if (ans != ErrCode::SUC) { + LOG(ERROR) << "Deal monitor request failed, because" << IPC_ERROR(ans); + } + } +} + +void DynoLogNpuMonitor::Finalize() +{ + msptiMonitor_.Uninit(); +} +} // namespace ipc_monitor +} // namespace dynolog_npu diff --git a/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.h b/msmonitor/plugin/ipc_monitor/DynoLogNpuMonitor.h similarity index 62% rename from dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.h rename to msmonitor/plugin/ipc_monitor/DynoLogNpuMonitor.h index 40ee21072710312a86cd75befdcefa67e24efb8f..07345d75de496b59b42e09ead90d77747671930f 100644 --- a/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.h +++ b/msmonitor/plugin/ipc_monitor/DynoLogNpuMonitor.h @@ -3,7 +3,9 @@ #include "MonitorBase.h" #include "NpuIpcClient.h" +#include "MsptiMonitor.h" #include "singleton.h" +#include "InputParser.h" namespace dynolog_npu { namespace ipc_monitor { @@ -12,22 +14,30 @@ class DynoLogNpuMonitor : public MonitorBase, public Singleton; public: - DynoLogNpuMonitor() = default; + DynoLogNpuMonitor(); bool Init() override; + ErrCode DealMonitorReq(MsptiMonitorCfg& cmd); std::string Poll() override; + void EnableMsptiMonitor(std::unordered_map& cfg_map); + void Finalize(); void SetNpuId(int id) override { npuId_ = id; } + IpcClient *GetIpcClient() + { + return &ipcClient_; + } + private: bool isInitialized_ = false; int32_t npuId_ = 0; IpcClient ipcClient_; + MsptiMonitor msptiMonitor_; }; } // namespace ipc_monitor } // namespace dynolog_npu -#endif - +#endif // DYNOLOG_NPU_MONITOR_H diff --git a/msmonitor/plugin/ipc_monitor/InputParser.cpp b/msmonitor/plugin/ipc_monitor/InputParser.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bc77d33f1ae2f029e2fe548f8f9d9a7f5a594935 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/InputParser.cpp @@ -0,0 +1,61 @@ +#include "InputParser.h" +#include +#include +#include "utils.h" + +namespace dynolog_npu { +namespace ipc_monitor { + +const std::string MSPTI_ACTIVITY_KIND_KEY = "MSPTI_ACTIVITY_KIND"; +const std::string REPORT_INTERVAL_S_KEY = "REPORT_INTERVAL_S"; +const std::string NPU_MONITOR_START_KEY = "NPU_MONITOR_START"; +const std::string NPU_MONITOR_STOP_KEY = "NPU_MONITOR_STOP"; + +const std::unordered_set cfgMap { + "MSPTI_ACTIVITY_KIND", + "REPORT_INTERVAL_S", + "NPU_MONITOR_START", + "NPU_MONITOR_STOP", + "REQUEST_TRACE_ID" +}; + +const std::unordered_map kindStrMap { + {"Marker", MSPTI_ACTIVITY_KIND_MARKER}, + {"Kernel", MSPTI_ACTIVITY_KIND_KERNEL}, + {"API", MSPTI_ACTIVITY_KIND_API}, + {"Hccl", MSPTI_ACTIVITY_KIND_HCCL}, + {"Memory", MSPTI_ACTIVITY_KIND_MEMORY}, + {"MemSet", MSPTI_ACTIVITY_KIND_MEMSET}, + {"MemCpy", MSPTI_ACTIVITY_KIND_MEMCPY} +}; + +std::set str2Kinds(const std::string& kindStrs) +{ + std::set res; + auto kindStrList = split(kindStrs, ','); + for (auto& kindStr : kindStrList) { + auto kind = kindStrMap.find(kindStr); + if (kind == kindStrMap.end()) { + return {MSPTI_ACTIVITY_KIND_INVALID}; + } + res.insert(kind->second); + } + return res; +} + +MsptiMonitorCfg InputParser::DynoLogGetOpts(std::unordered_map& cmd) +{ + if (cmd.count("NPU_MONITOR_SRART")) { + return {{MSPTI_ACTIVITY_KIND_INVALID}, 0, false, false, false}; + } + auto activityKinds = str2Kinds(cmd[MSPTI_ACTIVITY_KIND_KEY]); + uint32_t reportTimes = 0; + Str2Uint32(reportTimes, cmd[REPORT_INTERVAL_S_KEY]); + bool startSwitch = false; + Str2Bool(startSwitch, cmd[NPU_MONITOR_START_KEY]); + bool endSwitch = false; + Str2Bool(endSwitch, cmd[NPU_MONITOR_STOP_KEY]); + return {activityKinds, reportTimes, startSwitch, endSwitch, true}; +} +} +} \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/InputParser.h b/msmonitor/plugin/ipc_monitor/InputParser.h new file mode 100644 index 0000000000000000000000000000000000000000..e5f674e1605b3721a75372113ee5d7f012c5e506 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/InputParser.h @@ -0,0 +1,30 @@ +#ifndef INPUT_PARSER_H +#define INPUT_PARSER_H + +#include +#include +#include +#include + +namespace dynolog_npu { +namespace ipc_monitor { + +struct MsptiMonitorCfg +{ + std::set enableActivities; + uint32_t reportIntervals; + bool monitorStart; + bool monitorStop; + bool isMonitor; +}; + + +class InputParser: public dynolog_npu::ipc_monitor::Singleton { +public: + MsptiMonitorCfg DynoLogGetOpts(std::unordered_map& cmd); +}; + +} // namespace ipc_monitor +} // namespace dynolog_npu + +#endif \ No newline at end of file diff --git a/dynolog_npu/plugin/ipc_monitor/MonitorBase.h b/msmonitor/plugin/ipc_monitor/MonitorBase.h similarity index 92% rename from dynolog_npu/plugin/ipc_monitor/MonitorBase.h rename to msmonitor/plugin/ipc_monitor/MonitorBase.h index 108023c7624b747e5987be9184d6c594decd360a..29be0b6be04083babb8d20e5386e93c053a41357 100644 --- a/dynolog_npu/plugin/ipc_monitor/MonitorBase.h +++ b/msmonitor/plugin/ipc_monitor/MonitorBase.h @@ -1,5 +1,6 @@ #ifndef MONITOR_BASE_H #define MONITOR_BASE_H + #include namespace dynolog_npu { @@ -14,5 +15,4 @@ public: } // namespace ipc_monitor } // namespace dynolog_npu - -#endif \ No newline at end of file +#endif // MONITOR_BASE_H diff --git a/dynolog_npu/plugin/ipc_monitor/NpuIpcClient.cpp b/msmonitor/plugin/ipc_monitor/NpuIpcClient.cpp similarity index 75% rename from dynolog_npu/plugin/ipc_monitor/NpuIpcClient.cpp rename to msmonitor/plugin/ipc_monitor/NpuIpcClient.cpp index ca2429f1e368ad996b8a8a954810ed7439c78bea..d9a5c33c9e359bec3386439450877074e5199e2b 100644 --- a/dynolog_npu/plugin/ipc_monitor/NpuIpcClient.cpp +++ b/msmonitor/plugin/ipc_monitor/NpuIpcClient.cpp @@ -1,8 +1,13 @@ #include "NpuIpcClient.h" - +#include namespace dynolog_npu { namespace ipc_monitor { +bool IpcClient::Init() +{ + pids_ = GetPids(); + return true; +} bool IpcClient::RegisterInstance(int32_t id) { @@ -11,46 +16,52 @@ bool IpcClient::RegisterInstance(int32_t id) .pid = getpid(), .jobId = JOB_ID, }; - std::unique_ptr message = Message::ConstructMessage(context, "ctxt"); + std::unique_ptr message = Message::ConstructMessage(context, MSG_TYPE_CONTEXT); try { - if (!SyncSendMessage(*message, std::string(DYNO_IPC_NAME))) { - LOG(ERROR) << "Failed to send register ctxt for pid " << context.pid << " with dyno"; + if (!SyncSendMessage(*message, DYNO_IPC_NAME)) { + LOG(WARNING) << "Failed to send register ctxt for pid " << context.pid << " with dyno"; return false; } } catch (const std::exception &e) { - LOG(ERROR) << " Error when SyncSendMessage: " << e.what(); + LOG(WARNING) << "Error when SyncSendMessage: " << e.what(); return false; } - LOG(INFO) << "Resigter pid " << context.pid << " for dynolog success !"; + LOG(INFO) << "Resigter pid " << context.pid << " for dynolog success!"; return true; } + std::string IpcClient::IpcClientNpuConfig() { auto size = pids_.size(); - auto *req = (NpuRequest *)malloc(sizeof(NpuRequest) + sizeof(int32_t) * size); + auto *req = ReinterpretConvert(malloc(sizeof(NpuRequest) + sizeof(int32_t) * size)); + if (req == nullptr) { + LOG(ERROR) << " Malloc for NpuRequest failed !"; + return ""; + } req->type = DYNO_IPC_TYPE; req->pidSize = size; req->jobId = JOB_ID; - for (int i = 0; i < size; i++) { + for (size_t i = 0; i < size; i++) { req->pids[i] = pids_[i]; } - std::unique_ptr message = Message::ConstructMessage(*req, "req", size); - if (!SyncSendMessage(*message, std::string(DYNO_IPC_NAME))) { - LOG(ERROR) << " Failed to send config to dyno server fail !"; + std::unique_ptr message = Message::ConstructMessage(*req, MSG_TYPE_REQUEST, size); + if (!SyncSendMessage(*message, DYNO_IPC_NAME)) { + LOG(WARNING) << "Failed to send config to dyno server"; free(req); req = nullptr; return ""; } free(req); + req = nullptr; message = PollRecvMessage(MAX_IPC_RETRIES, MAX_SLEEP_US); if (!message) { - LOG(ERROR) << " Failed to receive on-demand config !"; + LOG(WARNING) << "Failed to receive on-demand config"; return ""; } std::string res = std::string(ReinterpretConvert(message->buf.get()), message->metadata.size); - return res; } + std::unique_ptr IpcClient::ReceiveMessage() { std::lock_guard wguard(dequeLock_); @@ -61,10 +72,11 @@ std::unique_ptr IpcClient::ReceiveMessage() msgDynoDeque_.pop_front(); return message; } + bool IpcClient::SyncSendMessage(const Message &message, const std::string &destName, int numRetry, int seepTimeUs) { if (destName.empty()) { - LOG(ERROR) << " Can not send to empty socket name !"; + LOG(WARNING) << "Can not send to empty socket name!"; return false; } int i = 0; @@ -78,11 +90,12 @@ bool IpcClient::SyncSendMessage(const Message &message, const std::string &destN seepTimeUs *= 2; // 2: double sleep time } } catch (const std::exception &e) { - LOG(ERROR) << " Error when SyncSendMessage: " << e.what(); + LOG(ERROR) << "Error when SyncSendMessage: " << e.what(); return false; } return i < numRetry; } + bool IpcClient::Recv() { try { @@ -93,7 +106,7 @@ bool IpcClient::Recv() try { successFlag = ep_.TryPeekMessage(*peekCtxt); } catch (std::exception &e) { - LOG(ERROR) << " Error when TryPeekMessage: " << e.what(); + LOG(ERROR) << "Error when TryPeekMessage: " << e.what(); return false; } if (successFlag) { @@ -107,7 +120,7 @@ bool IpcClient::Recv() try { successFlag = ep_.TryRcvMessage(*recvCtxt); } catch (std::exception &e) { - LOG(ERROR) << " Error when TryRecvMsg: " << e.what(); + LOG(ERROR) << "Error when TryRecvMsg: " << e.what(); return false; } if (successFlag) { @@ -117,11 +130,12 @@ bool IpcClient::Recv() } } } catch (std::exception &e) { - LOG(ERROR) << " Error in Recv(): " << e.what(); + LOG(ERROR) << "Error in Recv(): " << e.what(); return false; } return false; } + std::unique_ptr IpcClient::PollRecvMessage(int maxRetry, int sleeTimeUs) { for (int i = 0; i < maxRetry; i++) { @@ -132,6 +146,5 @@ std::unique_ptr IpcClient::PollRecvMessage(int maxRetry, int sleeTimeUs } return nullptr; } - } // namespace ipc_monitor -} // namespace dynolog_npu \ No newline at end of file +} // namespace dynolog_npu diff --git a/dynolog_npu/plugin/ipc_monitor/NpuIpcClient.h b/msmonitor/plugin/ipc_monitor/NpuIpcClient.h similarity index 55% rename from dynolog_npu/plugin/ipc_monitor/NpuIpcClient.h rename to msmonitor/plugin/ipc_monitor/NpuIpcClient.h index ae7b00eb51b935db4e799fab470c3343e78bcb6f..5a2e55a8681b97d77ce03e76b28827d1e339880b 100644 --- a/dynolog_npu/plugin/ipc_monitor/NpuIpcClient.h +++ b/msmonitor/plugin/ipc_monitor/NpuIpcClient.h @@ -1,40 +1,44 @@ #ifndef NPU_IPC_CLIENT_H #define NPU_IPC_CLIENT_H -#include -#include + +#include #include #include -#include -#include -#include -#include #include "NpuIpcEndPoint.h" #include "utils.h" +#include "securec.h" namespace dynolog_npu { namespace ipc_monitor { constexpr int TYPE_SIZE = 32; constexpr int JOB_ID = 0; -constexpr const char *DYNO_IPC_NAME = "dynolog"; constexpr const int DYNO_IPC_TYPE = 3; constexpr const int MAX_IPC_RETRIES = 5; constexpr const int MAX_SLEEP_US = 10000; +const std::string DYNO_IPC_NAME = "dynolog"; +const std::string MSG_TYPE_REQUEST = "req"; +const std::string MSG_TYPE_CONTEXT = "ctxt"; +const std::string MSG_TYPE_DATA = "data"; + struct NpuRequest { int type; int pidSize; int64_t jobId; int32_t pids[0]; }; + struct NpuContext { int32_t npu; pid_t pid; int64_t jobId; }; + struct Metadata { size_t size = 0; char type[TYPE_SIZE] = ""; }; + struct Message { Metadata metadata; std::unique_ptr buf; @@ -45,19 +49,26 @@ struct Message { if (type.size() + 1 > sizeof(ipcNpuMessage->metadata.type)) { throw std::runtime_error("Type string is too long to fit in metadata.type" + IPC_ERROR(ErrCode::PARAM)); } - memcpy(ipcNpuMessage->metadata.type, type.c_str(), type.size() + 1); + if (memcpy_s(ipcNpuMessage->metadata.type, sizeof(ipcNpuMessage->metadata.type), + type.c_str(), type.size() + 1) != EOK) { + throw std::runtime_error("memcpy_s failed" + IPC_ERROR(ErrCode::MEMORY)); + } #if __cplusplus >= 201703L if constexpr (std::is_same::value == true) { ipcNpuMessage->metadata.size = data.size(); ipcNpuMessage->buf = std::make_unique(ipcNpuMessage->metadata.size); - memcpy(ipcNpuMessage->buf.get(), data.c_str(), sizeof(data)); + if (memcpy_s(ipcNpuMessage->buf.get(), ipcNpuMessage->metadata.size, data.c_str(), data.size()) != EOK) { + throw std::runtime_error("memcpy_s failed" + IPC_ERROR(ErrCode::MEMORY)); + } return ipcNpuMessage; } #endif static_assert(std::is_trivially_copyable::value); ipcNpuMessage->metadata.size = sizeof(data); ipcNpuMessage->buf = std::make_unique(ipcNpuMessage->metadata.size); - memcpy(ipcNpuMessage->buf.get(), &data, sizeof(data)); + if (memcpy_s(ipcNpuMessage->buf.get(), ipcNpuMessage->metadata.size, &data, sizeof(data)) != EOK) { + throw std::runtime_error("memcpy_s failed" + IPC_ERROR(ErrCode::MEMORY)); + } return ipcNpuMessage; } @@ -68,36 +79,61 @@ struct Message { if (type.size() + 1 > sizeof(ipcNpuMessage->metadata.type)) { throw std::runtime_error("Type string is too long to fit in metadata.type" + IPC_ERROR(ErrCode::PARAM)); } - memcpy(ipcNpuMessage->metadata.type, type.c_str(), type.size() + 1); + if (memcpy_s(ipcNpuMessage->metadata.type, sizeof(ipcNpuMessage->metadata.type), + type.c_str(), type.size() + 1) != EOK) { + throw std::runtime_error("memcpy_s failed" + IPC_ERROR(ErrCode::MEMORY)); + } static_assert(std::is_trivially_copyable::value); static_assert(std::is_trivially_copyable::value); ipcNpuMessage->metadata.size = sizeof(data) + sizeof(U) * n; ipcNpuMessage->buf = std::make_unique(ipcNpuMessage->metadata.size); - memcpy(ipcNpuMessage->buf.get(), &data, ipcNpuMessage->metadata.size); + if (memcpy_s(ipcNpuMessage->buf.get(), ipcNpuMessage->metadata.size, + &data, ipcNpuMessage->metadata.size) != EOK) { + throw std::runtime_error("memcpy_s failed" + IPC_ERROR(ErrCode::MEMORY)); + } + return ipcNpuMessage; + } + + static std::unique_ptr ConstructStrMessage(const std::string &data, const std::string &type) + { + std::unique_ptr ipcNpuMessage = std::make_unique(Message()); + if (type.size() + 1 > sizeof(ipcNpuMessage->metadata.type)) { + throw std::runtime_error("Type string is too long to fit in metadata.type" + IPC_ERROR(ErrCode::PARAM)); + } + if (memcpy_s(ipcNpuMessage->metadata.type, sizeof(ipcNpuMessage->metadata.type), + type.c_str(), type.size() + 1) != EOK) { + throw std::runtime_error("memcpy_s failed" + IPC_ERROR(ErrCode::MEMORY)); + } + ipcNpuMessage->metadata.size = data.size(); + ipcNpuMessage->buf = std::make_unique(ipcNpuMessage->metadata.size); + if (memcpy_s(ipcNpuMessage->buf.get(), ipcNpuMessage->metadata.size, data.c_str(), data.size()) != EOK) { + throw std::runtime_error("memcpy_s failed" + IPC_ERROR(ErrCode::MEMORY)); + } return ipcNpuMessage; } }; + class IpcClient { public: IpcClient(const IpcClient &) = delete; IpcClient &operator = (const IpcClient &) = delete; IpcClient() = default; + bool Init(); bool RegisterInstance(int32_t npu); std::string IpcClientNpuConfig(); + bool SyncSendMessage(const Message &message, const std::string &destName, int numRetry = 10, + int seepTimeUs = 10000); private: - std::vector pids_ = GetPids(); + std::vector pids_; NpuIpcEndPoint<0> ep_{ "dynoconfigclient" + GenerateUuidV4() }; std::mutex dequeLock_; std::deque> msgDynoDeque_; std::unique_ptr ReceiveMessage(); - bool SyncSendMessage(const Message &message, const std::string &destName, int numRetry = 10, - int seepTimeUs = 10000); bool Recv(); std::unique_ptr PollRecvMessage(int maxRetry, int sleeTimeUs); }; - } // namespace ipc_monitor } // namespace dynolog_npu -#endif \ No newline at end of file +#endif // NPU_IPC_CLIENT_H diff --git a/dynolog_npu/plugin/ipc_monitor/NpuIpcEndPoint.h b/msmonitor/plugin/ipc_monitor/NpuIpcEndPoint.h similarity index 84% rename from dynolog_npu/plugin/ipc_monitor/NpuIpcEndPoint.h rename to msmonitor/plugin/ipc_monitor/NpuIpcEndPoint.h index 6560fa515646226ddbffbca49c4f818eb0d0ebcf..ea6fe3f662261c12cc5bfda699756b8382542d37 100644 --- a/dynolog_npu/plugin/ipc_monitor/NpuIpcEndPoint.h +++ b/msmonitor/plugin/ipc_monitor/NpuIpcEndPoint.h @@ -1,16 +1,15 @@ #ifndef NPU_IPC_ENDPOINT_H #define NPU_IPC_ENDPOINT_H -#include + #include #include #include #include #include +#include #include -#include -#include -#include #include "utils.h" +#include "securec.h" namespace dynolog_npu { namespace ipc_monitor { @@ -46,23 +45,34 @@ public: if (socketFd == -1) { throw std::runtime_error(std::strerror(errno) + IPC_ERROR(ErrCode::PARAM)); } + int ret = 0; struct sockaddr_un address; size_t addressLen = SetSocketAdress(addressName, address); if (address.sun_path[0] != STR_END_CHAR) { - unlink(address.sun_path); + ret = unlink(address.sun_path); + } + if (ret == -1) { + throw std::runtime_error("Unlink failed, error is " + std::string(strerror(errno)) + IPC_ERROR(ErrCode::PARAM)); } - int res = bind(socketFd, ReinterpretConvert(&address), addressLen); - if (res == -1) { + + ret = bind(socketFd, ReinterpretConvert(&address), addressLen); + if (ret == -1) { throw std::runtime_error("Bind socket failed." + IPC_ERROR(ErrCode::PARAM)); } + if (address.sun_path[0] != STR_END_CHAR) { - chmod(address.sun_path, SOCKET_FD_CHMOD); + ret = chmod(address.sun_path, SOCKET_FD_CHMOD); + } + if (ret == -1) { + throw std::runtime_error("Chmod failed, error is " + std::string(strerror(errno)) + IPC_ERROR(ErrCode::PARAM)); } } + ~NpuIpcEndPoint() { close(socketFd); } + [[nodiscard]] auto BuildSendNpuCtxt(const std::string &desAddrName, const std::vector &npuPayLoad, const std::vector &fileDes) { @@ -80,7 +90,11 @@ public: throw std::runtime_error("Memcpy failed when fileDes size large than ctxt fileDesPtr " + IPC_ERROR(ErrCode::PARAM)); } - memcpy(ctxt->fileDesPtr, fileDes.data(), fileDes.size() * sizeof(fileDesT)); + if (memcpy_s(ctxt->fileDesPtr, sizeof(ctxt->fileDesPtr), + fileDes.data(), fileDes.size() * sizeof(fileDesT)) != EOK) { + throw std::runtime_error("Memcpy failed when fileDes size large than ctxt fileDesPtr " + + IPC_ERROR(ErrCode::MEMORY)); + } } return ctxt; } @@ -137,7 +151,7 @@ public: throw std::runtime_error("TryPeekMessage occur " + std::string(std::strerror(errno))); } - const char *GetName(Ctxt const & ctxt) const noexcept + const char *GetName(Ctxt const & ctxt) const { if (ctxt.messageName.sun_path[0] != STR_END_CHAR) { throw std::runtime_error("GetName() want to got abstract socket, but got " + @@ -173,8 +187,10 @@ protected: auto BuildNpuCtxt_(const std::vector &npuPayLoad, unsigned numFileDes) { auto ctxt = std::make_unique(npuPayLoad.size()); - std::memset(&ctxt->msghdr, 0, sizeof(ctxt->msghdr)); - for (auto i = 0; i < npuPayLoad.size(); i++) { + if (memset_s(&ctxt->msghdr, sizeof(ctxt->msghdr), 0, sizeof(ctxt->msghdr)) != EOK) { + throw std::runtime_error("Memset failed when build ctxt " + IPC_ERROR(ErrCode::MEMORY)); + } + for (size_t i = 0; i < npuPayLoad.size(); i++) { ctxt->iov[i] = {npuPayLoad[i].data, npuPayLoad[i].size}; } ctxt->msghdr.msg_name = &ctxt->messageName; @@ -197,8 +213,7 @@ protected: return ctxt; } }; - } // namespace ipc_monitor } // namespace dynolog_npu -#endif +#endif // NPU_IPC_ENDPOINT_H diff --git a/dynolog_npu/plugin/ipc_monitor/PyDynamicMonitorProxy.h b/msmonitor/plugin/ipc_monitor/PyDynamicMonitorProxy.h similarity index 67% rename from dynolog_npu/plugin/ipc_monitor/PyDynamicMonitorProxy.h rename to msmonitor/plugin/ipc_monitor/PyDynamicMonitorProxy.h index 0471a70a3419eeeee2986d1d18710ee112c70313..1a778aa121e21538daf7a5893f7b7a93ffbc6047 100644 --- a/dynolog_npu/plugin/ipc_monitor/PyDynamicMonitorProxy.h +++ b/msmonitor/plugin/ipc_monitor/PyDynamicMonitorProxy.h @@ -2,7 +2,6 @@ #define PYDYNAMIC_MONITOR_PROXY_H #include -#include #include "MonitorBase.h" #include "DynoLogNpuMonitor.h" @@ -15,15 +14,9 @@ public: bool InitDyno(int npuId) { try { - if (!google::IsGoogleLoggingInitialized()) { - google::InitGoogleLogging("DynoLogNpuMonitor"); - google::SetLogDestination(google::GLOG_INFO, "/var/log/dynolog_npu_"); - google::SetLogFilenameExtension(".log"); - } monitor_ = DynoLogNpuMonitor::GetInstance(); monitor_->SetNpuId(npuId); bool res = monitor_->Init(); - LOG(ERROR) << res; return res; } catch (const std::exception &e) { LOG(ERROR) << "Error when init dyno " << e.what(); @@ -33,14 +26,22 @@ public: std::string PollDyno() { - return monitor_->Poll(); - }; + return monitor_->Poll(); + } + + void EnableMsptiMonitor(std::unordered_map& config_map) + { + DynoLogNpuMonitor::GetInstance()->EnableMsptiMonitor(config_map); + } + void FinalizeDyno() + { + DynoLogNpuMonitor::GetInstance()->Finalize(); + } private: MonitorBase *monitor_ = nullptr; }; } // namespace ipc_monitor } // namespace dynolog_npu - -#endif +#endif // PYDYNAMIC_MONITOR_PROXY_H diff --git a/msmonitor/plugin/ipc_monitor/TimerTask.h b/msmonitor/plugin/ipc_monitor/TimerTask.h new file mode 100644 index 0000000000000000000000000000000000000000..d15d7e71c67ded93d1d131ed679da9ce636a73c8 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/TimerTask.h @@ -0,0 +1,103 @@ +#ifndef TIMER_TASK_H +#define TIMER_TASK_H + +#include +#include +#include +#include +#include +#include + +namespace dynolog_npu { +namespace ipc_monitor { +class TimerTask { +public: + TimerTask(const std::string& name, int interval) + : interval(interval), name(name), manual_trigger(false), running(false) {} + + ~TimerTask() + { + Stop(); + } + + void Run() + { + if (running) { + LOG(ERROR) << name << " Timer task is already running."; + return; + } + running = true; + taskThread = std::thread(&TimerTask::TaskRun, this); + } + + void Trigger() + { + std::unique_lock lock(cv_mutex); + manual_trigger = true; + if (running.load()) { + cv.notify_one(); + } + } + + // 停止定时任务 + void Stop() + { + if (!running) { + LOG(ERROR) << name << "Timer task is not running."; + return; + } + + running = false; + cv.notify_one(); + if (taskThread.joinable()) { + taskThread.join(); + } + } + + void SetInterval(int intervalTimes) + { + interval.store(intervalTimes); + } + + virtual void InitResource() {}; + virtual void ReleaseResource() {}; + virtual void ExecuteTask() = 0; +private: + // 定时任务线程函数 + void TaskRun() + { + LOG(INFO) << name << " Timer task started."; + InitResource(); + while (running) { + std::unique_lock lock(cv_mutex); + if (interval.load()) { + cv.wait_for(lock, std::chrono::seconds(interval.load()), [&] {return manual_trigger || !running;}); + } else { + cv.wait(lock, [&] {return manual_trigger || !running;}); + } + if (!running) { + break; + } + if (manual_trigger) { + manual_trigger = false; + } + if (running) { + ExecuteTask(); + } + } + ReleaseResource(); + LOG(INFO) << name << " Timer task stopped."; + } + + std::atomic interval; + std::string name; + std::condition_variable cv; + std::mutex cv_mutex; + std::atomic manual_trigger; + std::atomic running; + std::thread taskThread; +}; + +} +} +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricApiProcess.cpp b/msmonitor/plugin/ipc_monitor/metric/MetricApiProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..200ded4ee8530a5c8b3d14c242ef31e2312b5724 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricApiProcess.cpp @@ -0,0 +1,70 @@ +#include "MetricApiProcess.h" + +#include +#include + +#include "utils.h" + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +std::string ApiMetric::seriesToJson() +{ + nlohmann::json jsonMsg; + jsonMsg["kind"] = "API"; + jsonMsg["deviceId"] = -1; + jsonMsg["duration"] = duration; + jsonMsg["timestamp"] = timestamp; + return jsonMsg.dump(); +} + +void MetricApiProcess::ConsumeMsptiData(msptiActivity *record) +{ + msptiActivityApi* apiData = ReinterpretConvert(record); + msptiActivityApi* tmp = ReinterpretConvert(MsptiMalloc(sizeof(msptiActivityApi), ALIGN_SIZE)); + if (memcpy_s(tmp, sizeof(msptiActivityApi), apiData, sizeof(msptiActivityApi)) != EOK) { + MsptiFree(ReinterpretConvert(tmp)); + LOG(ERROR) << "memcpy_s failed" << IPC_ERROR(ErrCode::MEMORY); + return; + } + { + std::unique_lock lock(dataMutex); + records.emplace_back(tmp); + } +} + +std::vector MetricApiProcess::AggregatedData() +{ + std::vector> copyRecords; + { + std::unique_lock lock(dataMutex); + copyRecords = std::move(records); + records.clear(); + } + ApiMetric apiMetric{}; + auto ans = std::accumulate(copyRecords.begin(), copyRecords.end(), 0ULL, + [](uint64_t acc, std::shared_ptr api) { + return acc + api->end - api->start; + }); + apiMetric.duration = ans; + apiMetric.deviceId = -1; + apiMetric.timestamp = getCurrentTimestamp64(); + return {apiMetric}; +} + +void MetricApiProcess::SendProcessMessage() +{ + auto afterAggregated = AggregatedData(); + for (auto& metric: afterAggregated) { + SendMessage(metric.seriesToJson()); + } +} + +void MetricApiProcess::Clear() +{ + records.clear(); +} +} +} +} diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricApiProcess.h b/msmonitor/plugin/ipc_monitor/metric/MetricApiProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..6939f2a0d55cffd3a7998447001f3b9f7c704f0f --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricApiProcess.h @@ -0,0 +1,37 @@ +#ifndef METRIC_API_PROCESS_H +#define METRIC_API_PROCESS_H + +#include +#include +#include "MetricProcessBase.h" + + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +struct ApiMetric { + uint64_t duration; + uint64_t timestamp; + uint32_t deviceId; +public: + std::string seriesToJson(); +}; + +class MetricApiProcess: public MetricProcessBase +{ +public: + MetricApiProcess() = default; + void ConsumeMsptiData(msptiActivity *record) override; + std::vector AggregatedData(); + void SendProcessMessage() override; + void Clear() override; +private: + std::mutex dataMutex; + std::vector> records; +}; +} +} +} + +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricHcclProcess.cpp b/msmonitor/plugin/ipc_monitor/metric/MetricHcclProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..2b7e92fb16fd087d41a3b762798caef0bc8d9b01 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricHcclProcess.cpp @@ -0,0 +1,81 @@ +#include "MetricHcclProcess.h" +#include +#include +#include "utils.h" + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +std::string HcclMetric::seriesToJson() +{ + nlohmann::json jsonMsg; + jsonMsg["kind"] = "Hccl"; + jsonMsg["deviceId"] = deviceId; + jsonMsg["duration"] = duration; + jsonMsg["timestamp"] = timestamp; + return jsonMsg.dump(); +} + +void MetricHcclProcess::ConsumeMsptiData(msptiActivity *record) +{ + msptiActivityHccl* hcclData = ReinterpretConvert(record); + msptiActivityHccl* tmp = ReinterpretConvert(MsptiMalloc(sizeof(msptiActivityHccl), ALIGN_SIZE)); + if (memcpy_s(tmp, sizeof(msptiActivityHccl), hcclData, sizeof(msptiActivityHccl)) != EOK) { + MsptiFree(ReinterpretConvert(tmp)); + LOG(ERROR) << "memcpy_s failed" << IPC_ERROR(ErrCode::MEMORY); + return; + } + { + std::unique_lock lock(dataMutex); + records.emplace_back(tmp); + } +} + +std::vector MetricHcclProcess::AggregatedData() +{ + std::vector> copyRecords; + { + std::unique_lock lock(dataMutex); + copyRecords = std::move(records); + records.clear(); + } + if (copyRecords.empty()) { + return {}; + } + std::unordered_map>> deviceId2HcclData = + groupby(copyRecords, [](const std::shared_ptr& data) -> std::uint32_t { + return data->ds.deviceId; + }); + std::vector ans; + auto curTimestamp = getCurrentTimestamp64(); + for (auto& pair: deviceId2HcclData) { + HcclMetric hcclMetric{}; + auto& hcclDatas = pair.second; + hcclMetric.duration = std::accumulate(hcclDatas.begin(), hcclDatas.end(), 0ULL, + [](uint64_t acc, std::shared_ptr hccl) { + return acc + hccl->end - hccl->start; + }); + hcclMetric.deviceId = pair.first; + hcclMetric.timestamp = curTimestamp; + ans.emplace_back(hcclMetric); + } + return ans; + +} + +void MetricHcclProcess::SendProcessMessage() +{ + auto afterAggregated = AggregatedData(); + for (auto& metric: afterAggregated) { + SendMessage(metric.seriesToJson()); + } +} + +void MetricHcclProcess::Clear() +{ + records.clear(); +} +} +} +} diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricHcclProcess.h b/msmonitor/plugin/ipc_monitor/metric/MetricHcclProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..d3753cca1e98bb8b6f80076a29419dc36d3cd1ad --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricHcclProcess.h @@ -0,0 +1,38 @@ +#ifndef METRIC_HCCL_PROCESS_H +#define METRIC_HCCL_PROCESS_H + +#include +#include +#include "MetricProcessBase.h" + + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +struct HcclMetric { + std::string kindName; + uint64_t duration; + uint64_t timestamp; + uint32_t deviceId; +public: + std::string seriesToJson(); +}; + +class MetricHcclProcess: public MetricProcessBase +{ +public: + MetricHcclProcess() = default; + void ConsumeMsptiData(msptiActivity *record) override; + std::vector AggregatedData(); + void SendProcessMessage() override; + void Clear() override; +private: + std::mutex dataMutex; + std::vector> records; +}; +} +} +} + +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricKernelProcess.cpp b/msmonitor/plugin/ipc_monitor/metric/MetricKernelProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5eef485b5a60b30d6a01c0779ab9e17b561d1ac4 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricKernelProcess.cpp @@ -0,0 +1,81 @@ +#include "MetricKernelProcess.h" + +#include + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +std::string KernelMetric::seriesToJson() +{ + nlohmann::json jsonMsg; + jsonMsg["kind"] = "Kernel"; + jsonMsg["deviceId"] = deviceId; + jsonMsg["duration"] = duration; + jsonMsg["timestamp"] = timestamp; + return jsonMsg.dump(); +} + +void MetricKernelProcess::ConsumeMsptiData(msptiActivity *record) +{ + msptiActivityKernel* kernel = ReinterpretConvert(record); + msptiActivityKernel* ptr = ReinterpretConvert(MsptiMalloc(sizeof(msptiActivityKernel), ALIGN_SIZE)); + if (memcpy_s(ptr, sizeof(msptiActivityKernel), kernel, sizeof(msptiActivityKernel)) != EOK) { + MsptiFree(ReinterpretConvert(ptr)); + LOG(ERROR) << "memcpy_s failed" << IPC_ERROR(ErrCode::MEMORY); + return; + } + { + std::unique_lock lock(dataMutex); + records.emplace_back(ptr); + } +} + +std::vector MetricKernelProcess::AggregatedData() +{ + std::vector> copyRecords; + { + std::unique_lock lock(dataMutex); + copyRecords = std::move(records); + records.clear(); + } + if (copyRecords.empty()) { + return {}; + } + std::unordered_map>> deviceId2KernelData = + groupby(copyRecords, [](const std::shared_ptr& data) -> std::uint32_t { + return data->ds.deviceId; + }); + std::vector ans; + auto curTimestamp = getCurrentTimestamp64(); + for (auto& pair: deviceId2KernelData) { + auto deviceId = pair.first; + auto& kernelDatas = pair.second; + KernelMetric kernelMetric{}; + kernelMetric.duration = std::accumulate(kernelDatas.begin(), kernelDatas.end(), 0ULL, + [](uint64_t acc, std::shared_ptr kernel) { + return acc + kernel->end - kernel->start; + }); + kernelMetric.deviceId = deviceId; + kernelMetric.timestamp = curTimestamp; + ans.emplace_back(kernelMetric); + } + + return ans; +} + +void MetricKernelProcess::SendProcessMessage() +{ + auto afterAggregated = AggregatedData(); + for (auto& metric: afterAggregated) { + SendMessage(metric.seriesToJson()); + } +} + +void MetricKernelProcess::Clear() +{ + records.clear(); +} +} +} +} diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricKernelProcess.h b/msmonitor/plugin/ipc_monitor/metric/MetricKernelProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..0107a26c283804002bd7ae7eab06e92c1a6ebbbf --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricKernelProcess.h @@ -0,0 +1,36 @@ +#ifndef METRIC_KERNEL_PROCESS_H +#define METRIC_KERNEL_PROCESS_H + +#include +#include "MetricProcessBase.h" + + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +struct KernelMetric { + uint64_t duration; + uint64_t timestamp; + uint32_t deviceId; +public: + std::string seriesToJson(); +}; + +class MetricKernelProcess: public MetricProcessBase +{ +public: + MetricKernelProcess() = default; + void ConsumeMsptiData(msptiActivity *record) override; + std::vector AggregatedData(); + void SendProcessMessage() override; + void Clear() override; +private: + std::mutex dataMutex; + std::vector> records; +}; +} +} +} + +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricManager.cpp b/msmonitor/plugin/ipc_monitor/metric/MetricManager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ca71065c36dff340567f95512258e7a540476fda --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricManager.cpp @@ -0,0 +1,78 @@ +#include "MetricManager.h" +#include "MetricKernelProcess.h" +#include "MetricApiProcess.h" +#include "MetricMemCpyProcess.h" +#include "MetricHcclProcess.h" +#include "MetricMarkProcess.h" +#include "MetricMemSetProcess.h" +#include "MetricMemProcess.h" +#include "utils.h" + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +MetricManager::MetricManager(): TimerTask("MetricManager", DEFAULT_FLUSH_INTERVAL), +kindSwitchs_(MSPTI_ACTIVITY_KIND_COUNT), consumeStatus_(MSPTI_ACTIVITY_KIND_COUNT){ + metrics.resize(MSPTI_ACTIVITY_KIND_COUNT); + metrics[MSPTI_ACTIVITY_KIND_KERNEL] = std::make_shared(); + metrics[MSPTI_ACTIVITY_KIND_API] = std::make_shared(); + metrics[MSPTI_ACTIVITY_KIND_MEMCPY] = std::make_shared(); + metrics[MSPTI_ACTIVITY_KIND_MARKER] = std::make_shared(); + metrics[MSPTI_ACTIVITY_KIND_MEMSET] = std::make_shared(); + metrics[MSPTI_ACTIVITY_KIND_HCCL] = std::make_shared(); + metrics[MSPTI_ACTIVITY_KIND_MEMORY] = std::make_shared(); +} + +void MetricManager::ReleaseResource() +{ + for (int i = 0; i < MSPTI_ACTIVITY_KIND_COUNT; i++) { + if (kindSwitchs_[i].load()) { + kindSwitchs_[i] = false; + metrics[i]->Clear(); + } + } +} + +ErrCode MetricManager::ConsumeMsptiData(msptiActivity *record) +{ + if (!kindSwitchs_[record->kind]) { + return ErrCode::PERMISSION; + } + auto metricProcess = metrics[record->kind]; + consumeStatus_[record->kind] = true; + metricProcess->ConsumeMsptiData(record); + consumeStatus_[record->kind] = false; + return ErrCode::SUC; +} + +void MetricManager::SetReportInterval(uint32_t intervalTimes) +{ + if (reportInterval_.load() != intervalTimes) { + SendMetricMsg(); + SetInterval(intervalTimes); + reportInterval_.store(intervalTimes); + } +} + +void MetricManager::ExecuteTask() +{ + SendMetricMsg(); +} + +void MetricManager::SendMetricMsg() +{ + for (int i = 0; i < MSPTI_ACTIVITY_KIND_COUNT; i++) { + if (kindSwitchs_[i].load()) { + metrics[i]->SendProcessMessage(); + } + } +} + +void MetricManager::EnableKindSwitch_(msptiActivityKind kind, bool flag) +{ + kindSwitchs_[kind] = flag; +} +} +} +} \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricManager.h b/msmonitor/plugin/ipc_monitor/metric/MetricManager.h new file mode 100644 index 0000000000000000000000000000000000000000..42b6d088fb382c0cef0aa1b19dbe1c1285babb51 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricManager.h @@ -0,0 +1,36 @@ +#ifndef METRIC_MANAGER_H +#define METRIC_MANAGER_H + +#include +#include + +#include "utils.h" +#include "singleton.h" +#include "mspti.h" +#include "TimerTask.h" +#include "MetricProcessBase.h" + +namespace dynolog_npu { +namespace ipc_monitor { +namespace metric { +class MetricManager: public ipc_monitor::Singleton, public TimerTask +{ +public: + MetricManager(); + ~MetricManager() = default; + ErrCode ConsumeMsptiData(msptiActivity *record); + void SetReportInterval(uint32_t intervalTimes); + void SendMetricMsg(); + void ExecuteTask() override; + void EnableKindSwitch_(msptiActivityKind kind, bool flag); + void ReleaseResource() override; +private: + std::vector> kindSwitchs_; + std::vector> consumeStatus_; + std::atomic reportInterval_; + std::vector> metrics; +}; +} +} +} +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricMarkProcess.cpp b/msmonitor/plugin/ipc_monitor/metric/MetricMarkProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d17b356b50864cd73dac70b32ab8f33e60349b35 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricMarkProcess.cpp @@ -0,0 +1,141 @@ +#include "MetricMarkProcess.h" + +#include +#include +#include + +#include "utils.h" + + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +constexpr size_t COMPLETE_RANGE_DATA_SIZE = 4; + +std::string MarkMetric::seriesToJson() +{ + nlohmann::json jsonMsg; + jsonMsg["kind"] = "Marker"; + jsonMsg["deviceId"] = deviceId; + jsonMsg["domain"] = domain; + jsonMsg["duration"] = duration; + jsonMsg["timestamp"] = timestamp; + return jsonMsg.dump(); +} + +bool MetricMarkProcess::TransMarkData2Range(const std::vector>& markDatas, + RangeMarkData& rangemarkData) { + if(markDatas.size() != COMPLETE_RANGE_DATA_SIZE) { + return false; + } + + for (auto& activityMarker: markDatas) { + if (activityMarker->flag == MSPTI_ACTIVITY_FLAG_MARKER_START_WITH_DEVICE) { + if (activityMarker->sourceKind == MSPTI_ACTIVITY_SOURCE_KIND_DEVICE) { + rangemarkData.deviceId = activityMarker->objectId.ds.deviceId; + rangemarkData.deviceStart = activityMarker->timestamp; + } else { + rangemarkData.start = activityMarker->timestamp; + } + } + if (activityMarker->flag == MSPTI_ACTIVITY_FLAG_MARKER_END_WITH_DEVICE) { + if (activityMarker->sourceKind == MSPTI_ACTIVITY_SOURCE_KIND_DEVICE) { + rangemarkData.deviceEnd = activityMarker->timestamp; + } else { + rangemarkData.end = activityMarker->timestamp; + } + } + } + auto markId = markDatas[0]->id; + std::string domainName = "default"; + auto it = domainMsg.find(markId); + if (it != domainMsg.end()) { + domainName = *it->second; + } + rangemarkData.domain = domainName; + id2Marker.erase(markId); + domainMsg.erase(markId); + return true; +} + +void MetricMarkProcess::ConsumeMsptiData(msptiActivity *record) +{ + msptiActivityMarker* markerData = ReinterpretConvert(record); + msptiActivityMarker* tmp = ReinterpretConvert(MsptiMalloc(sizeof(msptiActivityMarker), ALIGN_SIZE)); + if (memcpy_s(tmp, sizeof(msptiActivityMarker), markerData, sizeof(msptiActivityMarker)) != EOK) { + MsptiFree(ReinterpretConvert(tmp)); + LOG(ERROR) << "memcpy_s failed" << IPC_ERROR(ErrCode::MEMORY); + return; + } + { + std::unique_lock lock(dataMutex); + records.emplace_back(tmp); + if (markerData->flag == MSPTI_ACTIVITY_FLAG_MARKER_START_WITH_DEVICE && + markerData->sourceKind == MSPTI_ACTIVITY_SOURCE_KIND_HOST) { + std::string domainStr = markerData->domain; + auto markId = markerData->id; + domainMsg.emplace(markId, std::make_shared(domainStr)); + } + } +} + +std::vector MetricMarkProcess::AggregatedData() +{ + std::vector> copyRecords; + { + std::unique_lock lock(dataMutex); + copyRecords = std::move(records); + records.clear(); + } + for (auto& record: copyRecords) { + id2Marker[record->id].emplace_back(std::move(record)); + } + std::vector rangeDatas; + for (auto pair = id2Marker.rbegin(); pair != id2Marker.rend(); ++pair) { + auto markId = pair->first; + auto markDatas = pair->second; + RangeMarkData rangeMark{}; + if (TransMarkData2Range(markDatas, rangeMark)) { + rangeDatas.emplace_back(rangeMark); + } + } + + std::unordered_map> domain2RangeData = + groupby(rangeDatas, [](const RangeMarkData& data) -> std::string { + return data.domain + std::to_string(data.deviceId); + }); + std::vector ans; + for (auto& pair: domain2RangeData) { + MarkMetric markMetric{}; + auto domainName = pair.first; + auto rangeDatas = pair.second; + markMetric.deviceId = rangeDatas[0].deviceId; + markMetric.domain = domainName; + markMetric.timestamp = getCurrentTimestamp64(); + markMetric.duration = std::accumulate(rangeDatas.begin(), rangeDatas.end(), 0ULL, + [](uint64_t acc, const RangeMarkData& rangeData) { + return acc + rangeData.deviceEnd - rangeData.deviceStart; + }); + ans.emplace_back(markMetric); + } + return ans; +} + +void MetricMarkProcess::SendProcessMessage() +{ + auto afterAggregated = AggregatedData(); + for (auto& metric: afterAggregated) { + SendMessage(metric.seriesToJson()); + } +} + +void MetricMarkProcess::Clear() +{ + records.clear(); + domainMsg.clear(); + id2Marker.clear(); +} +} +} +} diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricMarkProcess.h b/msmonitor/plugin/ipc_monitor/metric/MetricMarkProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..63e08b58d43cee43630337aca8613345b6eee1a9 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricMarkProcess.h @@ -0,0 +1,56 @@ +#ifndef METRIC_MARK_PROCESS_H +#define METRIC_MARK_PROCESS_H + +#include +#include +#include "MetricProcessBase.h" + + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +struct MarkMetric { + std::string name; + std::string domain; + uint64_t duration; + uint64_t timestamp; + uint32_t deviceId; +public: + std::string seriesToJson(); +}; + +struct RangeMarkData +{ + std::string domain; + uint64_t duration; + uint64_t start{0}; + uint64_t end{0}; + uint64_t deviceStart{0}; + uint64_t deviceEnd{0}; + uint32_t deviceId; +}; + + +class MetricMarkProcess: public MetricProcessBase +{ +public: + MetricMarkProcess() = default; + void ConsumeMsptiData(msptiActivity *record) override; + std::vector AggregatedData(); + void SendProcessMessage() override; + void Clear() override; +private: + bool TransMarkData2Range(const std::vector>& markDatas, + RangeMarkData& rangemarkData); +private: + std::mutex dataMutex; + std::unordered_map> domainMsg; + std::vector> records; + std::map>> id2Marker; +}; +} +} +} + +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricMemCpyProcess.cpp b/msmonitor/plugin/ipc_monitor/metric/MetricMemCpyProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..77912d55c8baa87ea75438342ad8201af3c2efe6 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricMemCpyProcess.cpp @@ -0,0 +1,80 @@ +#include "MetricMemCpyProcess.h" + +#include + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +std::string MemCpyMetric::seriesToJson() +{ + nlohmann::json jsonMsg; + jsonMsg["kind"] = "MemCpy"; + jsonMsg["deviceId"] = deviceId; + jsonMsg["duration"] = duration; + jsonMsg["timestamp"] = timestamp; + return jsonMsg.dump(); +} + +void MetricMemCpyProcess::ConsumeMsptiData(msptiActivity *record) +{ + msptiActivityMemcpy* kernel = ReinterpretConvert(record); + msptiActivityMemcpy* ptr = ReinterpretConvert(MsptiMalloc(sizeof(msptiActivityMemcpy), ALIGN_SIZE)); + if (memcpy_s(ptr, sizeof(msptiActivityMemcpy), kernel, sizeof(msptiActivityMemcpy)) != EOK) { + MsptiFree(ReinterpretConvert(ptr)); + LOG(ERROR) << "memcpy_s failed" << IPC_ERROR(ErrCode::MEMORY); + return; + } + { + std::unique_lock lock(dataMutex); + records.emplace_back(ptr); + } +} + +std::vector MetricMemCpyProcess::AggregatedData() +{ + std::vector> copyRecords; + { + std::unique_lock lock(dataMutex); + copyRecords = std::move(records); + records.clear(); + } + if (copyRecords.empty()) { + return {}; + } + std::unordered_map>> deviceId2Memcpy = + groupby(copyRecords, [](const std::shared_ptr& data) -> std::uint32_t { + return data->deviceId; + }); + std::vector ans; + auto curTimestamp = getCurrentTimestamp64(); + for (auto& pair: deviceId2Memcpy) { + auto deviceId = pair.first; + MemCpyMetric memCpyMetric{}; + auto& memCpyDatas = pair.second; + memCpyMetric.duration = std::accumulate(memCpyDatas.begin(), memCpyDatas.end(), 0ULL, + [](uint64_t acc, std::shared_ptr memcpy) { + return acc + memcpy->end - memcpy->start; + }); + memCpyMetric.deviceId = deviceId; + memCpyMetric.timestamp = curTimestamp; + ans.emplace_back(memCpyMetric); + } + return ans; +} + +void MetricMemCpyProcess::SendProcessMessage() +{ + auto afterAggregated = AggregatedData(); + for (auto& metric: afterAggregated) { + SendMessage(metric.seriesToJson()); + } +} + +void MetricMemCpyProcess::Clear() +{ + records.clear(); +} +} +} +} diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricMemCpyProcess.h b/msmonitor/plugin/ipc_monitor/metric/MetricMemCpyProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..30ba8731923d9924a29f2145266c7d39cbcc0912 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricMemCpyProcess.h @@ -0,0 +1,36 @@ +#ifndef METRIC_MEMCPY_PROCESS_H +#define METRIC_MEMCPY_PROCESS_H + +#include +#include "MetricProcessBase.h" + + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +struct MemCpyMetric { + uint64_t duration; + uint64_t timestamp; + uint32_t deviceId; +public: + std::string seriesToJson(); +}; + +class MetricMemCpyProcess: public MetricProcessBase +{ +public: + MetricMemCpyProcess() = default; + void ConsumeMsptiData(msptiActivity *record) override; + std::vector AggregatedData(); + void SendProcessMessage() override; + void Clear() override; +private: + std::mutex dataMutex; + std::vector> records; +}; +} +} +} + +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricMemProcess.cpp b/msmonitor/plugin/ipc_monitor/metric/MetricMemProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a14faeb1be80bd988fda0984f2313fb967ea89a7 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricMemProcess.cpp @@ -0,0 +1,80 @@ +#include "MetricMemProcess.h" + +#include + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +std::string MemMetric::seriesToJson() +{ + nlohmann::json jsonMsg; + jsonMsg["kind"] = "Memory"; + jsonMsg["deviceId"] = deviceId; + jsonMsg["duration"] = duration; + jsonMsg["timestamp"] = timestamp; + return jsonMsg.dump(); +} + +void MetricMemProcess::ConsumeMsptiData(msptiActivity *record) +{ + msptiActivityMemory* mem = ReinterpretConvert(record); + msptiActivityMemory* ptr = ReinterpretConvert(MsptiMalloc(sizeof(msptiActivityMemory), ALIGN_SIZE)); + if (memcpy_s(ptr, sizeof(msptiActivityMemory), mem, sizeof(msptiActivityMemory)) != EOK) { + MsptiFree(ReinterpretConvert(ptr)); + LOG(ERROR) << "memcpy_s failed" << IPC_ERROR(ErrCode::MEMORY); + return; + } + { + std::unique_lock lock(dataMutex); + records.emplace_back(ptr); + } +} + +std::vector MetricMemProcess::AggregatedData() +{ + std::vector> copyRecords; + { + std::unique_lock lock(dataMutex); + copyRecords = std::move(records); + records.clear(); + } + if (copyRecords.empty()) { + return {}; + } + std::unordered_map>> deviceId2MemData = + groupby(copyRecords, [](const std::shared_ptr& data) -> std::uint32_t { + return data->deviceId; + }); + std::vector ans; + auto curTimestamp = getCurrentTimestamp64(); + for (auto& pair: deviceId2MemData) { + auto deviceId = pair.first; + auto& memDatas = pair.second; + MemMetric memMetric{}; + memMetric.duration = std::accumulate(memDatas.begin(), memDatas.end(), 0ULL, + [](uint64_t acc, std::shared_ptr mem) { + return acc + mem->end - mem->start; + }); + memMetric.deviceId = deviceId; + memMetric.timestamp = curTimestamp; + ans.emplace_back(memMetric); + } + return ans; +} + +void MetricMemProcess::SendProcessMessage() +{ + auto afterAggregated = AggregatedData(); + for (auto& metric: afterAggregated) { + SendMessage(metric.seriesToJson()); + } +} + +void MetricMemProcess::Clear() +{ + records.clear(); +} +} +} +} \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricMemProcess.h b/msmonitor/plugin/ipc_monitor/metric/MetricMemProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..c6193c89e729c4d07cdd26252ddf6b7004fb8ea0 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricMemProcess.h @@ -0,0 +1,37 @@ +#ifndef METRIC_MEM_PROCESS_H +#define METRIC_MEM_PROCESS_H + +#include +#include "MetricProcessBase.h" + + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +struct MemMetric { + std::string name; + uint64_t duration; + uint64_t timestamp; + uint32_t deviceId; +public: + std::string seriesToJson(); +}; + +class MetricMemProcess: public MetricProcessBase +{ +public: + MetricMemProcess() = default; + void ConsumeMsptiData(msptiActivity *record) override; + std::vector AggregatedData(); + void SendProcessMessage() override; + void Clear() override; +private: + std::mutex dataMutex; + std::vector> records; +}; +} +} +} + +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricMemSetProcess.cpp b/msmonitor/plugin/ipc_monitor/metric/MetricMemSetProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0fe539783e70020e20da17e1b8abd3f181b65be9 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricMemSetProcess.cpp @@ -0,0 +1,80 @@ +#include "MetricMemSetProcess.h" + +#include + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +std::string MemSetMetric::seriesToJson() +{ + nlohmann::json jsonMsg; + jsonMsg["kind"] = "MemSet"; + jsonMsg["deviceId"] = deviceId; + jsonMsg["duration"] = duration; + jsonMsg["timestamp"] = timestamp; + return jsonMsg.dump(); +} + +void MetricMemSetProcess::ConsumeMsptiData(msptiActivity *record) +{ + msptiActivityMemset* memSet = ReinterpretConvert(record); + msptiActivityMemset* ptr = ReinterpretConvert(MsptiMalloc(sizeof(msptiActivityMemset), ALIGN_SIZE)); + if (memcpy_s(ptr, sizeof(msptiActivityMemset), memSet, sizeof(msptiActivityMemset)) != EOK) { + MsptiFree(ReinterpretConvert(ptr)); + LOG(ERROR) << "memcpy_s failed" << IPC_ERROR(ErrCode::MEMORY); + return; + } + { + std::unique_lock lock(dataMutex); + records.emplace_back(ptr); + } +} + +std::vector MetricMemSetProcess::AggregatedData() +{ + std::vector> copyRecords; + { + std::unique_lock lock(dataMutex); + copyRecords = std::move(records); + records.clear(); + } + if (copyRecords.empty()) { + return {}; + } + std::unordered_map>> deviceId2MemsetData = + groupby(copyRecords, [](const std::shared_ptr& data) -> std::uint32_t { + return data->deviceId; + }); + std::vector ans; + auto curTimestamp = getCurrentTimestamp64(); + for (auto& pair: deviceId2MemsetData) { + MemSetMetric memSetMetric{}; + auto deviceId = pair.first; + auto& memSetDatas = pair.second; + memSetMetric.duration = std::accumulate(memSetDatas.begin(), memSetDatas.end(), 0ULL, + [](uint64_t acc, std::shared_ptr memSet) { + return acc + memSet->end - memSet->start; + }); + memSetMetric.deviceId = deviceId; + memSetMetric.timestamp = curTimestamp; + ans.emplace_back(memSetMetric); + } + return ans; +} + +void MetricMemSetProcess::SendProcessMessage() +{ + auto afterAggregated = AggregatedData(); + for (auto& metric: afterAggregated) { + SendMessage(metric.seriesToJson()); + } +} + +void MetricMemSetProcess::Clear() +{ + records.clear(); +} +} +} +} \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricMemSetProcess.h b/msmonitor/plugin/ipc_monitor/metric/MetricMemSetProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..c702a19c05c90d121278f4efeb569c975bf9e96c --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricMemSetProcess.h @@ -0,0 +1,37 @@ +#ifndef METRIC_MEM_SET_PROCESS_H +#define METRIC_MEM_SET_PROCESS_H + +#include +#include "metric/MetricProcessBase.h" + + +namespace dynolog_npu { +namespace ipc_monitor{ +namespace metric { + +struct MemSetMetric { + std::string name; + uint64_t duration; + uint64_t timestamp; + uint32_t deviceId; +public: + std::string seriesToJson(); +}; + +class MetricMemSetProcess: public MetricProcessBase +{ +public: + MetricMemSetProcess() = default; + void ConsumeMsptiData(msptiActivity *record) override; + std::vector AggregatedData(); + void SendProcessMessage() override; + void Clear() override; +private: + std::mutex dataMutex; + std::vector> records; +}; +} +} +} + +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/metric/MetricProcessBase.h b/msmonitor/plugin/ipc_monitor/metric/MetricProcessBase.h new file mode 100644 index 0000000000000000000000000000000000000000..1e74431c3ca8bcd920e748af96121c0d8551342c --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/metric/MetricProcessBase.h @@ -0,0 +1,46 @@ +#ifndef METRIC_PROCESS_BASE_H +#define METRIC_PROCESS_BASE_H + +#include +#include + +#include "DynoLogNpuMonitor.h" +#include "NpuIpcClient.h" +#include "mspti.h" + +namespace dynolog_npu { +namespace ipc_monitor { +namespace metric { +class MetricProcessBase +{ +public: + void SendMessage(std::string message) + { + if (message.empty()) { + LOG(ERROR) << "SendMessage message is empty"; + return; + } + static const std::string destName = DYNO_IPC_NAME + "_data"; + static const int maxRetry = 5, retryWaitTimeUs = 1000; + auto msg = Message::ConstructStrMessage(message, MSG_TYPE_DATA); + if (!msg) { + LOG(ERROR) << "ConstructStrMessage failed, message: " << message; + return; + } + auto ipcClient = DynoLogNpuMonitor::GetInstance()->GetIpcClient(); + if (!ipcClient) { + LOG(ERROR) << "DynoLogNpuMonitor ipcClient is nullptr"; + return; + } + if (!ipcClient->SyncSendMessage(*msg, destName, maxRetry, retryWaitTimeUs)) { + LOG(ERROR) << "send mspti message failed: " << message; + } + } + virtual void ConsumeMsptiData(msptiActivity *record) = 0; + virtual void Clear() = 0; + virtual void SendProcessMessage() = 0; +}; +} +} +} +#endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/mspti_monitor/MsptiMonitor.cpp b/msmonitor/plugin/ipc_monitor/mspti_monitor/MsptiMonitor.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bed07e3475ee9947396e3891d2240d95787feb38 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/mspti_monitor/MsptiMonitor.cpp @@ -0,0 +1,219 @@ +#include "MsptiMonitor.h" + +#include +#include +#include +#include + +#include "DynoLogNpuMonitor.h" +#include "MetricManager.h" +#include "utils.h" + +namespace { +constexpr size_t DEFAULT_BUFFER_SIZE = 8 * 1024 * 1024; +constexpr size_t MAX_BUFFER_SIZE = 256 * 1024 * 1024; +constexpr uint32_t MAX_ALLOC_CNT = MAX_BUFFER_SIZE / DEFAULT_BUFFER_SIZE; +} + +namespace dynolog_npu { +namespace ipc_monitor { + +MsptiMonitor::MsptiMonitor() + : start_(false), + subscriber_(nullptr), + checkFlush_(false), + flushInterval_(0) {} + +MsptiMonitor::~MsptiMonitor() +{ + Uninit(); +} + +void MsptiMonitor::Start() +{ + if (start_.load()) { + return; + } + SetThreadName("MsptiMonitor"); + if (Thread::Start() != 0) { + LOG(ERROR) << "MsptiMonitor start failed"; + return; + } + start_.store(true); + metric::MetricManager::GetInstance()->Run(); + LOG(INFO) << "MsptiMonitor start successfully"; +} + +void MsptiMonitor::Stop() +{ + if (!start_.load()) { + LOG(WARNING) << "MsptiMonitor is not running"; + return; + } + Uninit(); + if (msptiActivityFlushAll(1) != MSPTI_SUCCESS) { + LOG(WARNING) << "MsptiMonitor stop msptiActivityFlushAll failed"; + } + LOG(INFO) << "MsptiMonitor stop successfully"; +} + +void MsptiMonitor::Uninit() +{ + if (!start_.load()) { + return; + } + metric::MetricManager::GetInstance()->Stop(); + start_.store(false); + cv_.notify_one(); + Thread::Stop(); +} + +void MsptiMonitor::EnableActivity(msptiActivityKind kind) +{ + if (MSPTI_ACTIVITY_KIND_INVALID < kind && kind < MSPTI_ACTIVITY_KIND_COUNT) { + std::lock_guard lock(activityMtx_); + if (msptiActivityEnable(kind) == MSPTI_SUCCESS) { + enabledActivities_.insert(kind); + } else { + LOG(ERROR) << "MsptiMonitor enableActivity failed, kind: " << static_cast(kind); + } + metric::MetricManager::GetInstance()->EnableKindSwitch_(kind, true); + } +} + +void MsptiMonitor::DisableActivity(msptiActivityKind kind) +{ + if (MSPTI_ACTIVITY_KIND_INVALID < kind && kind < MSPTI_ACTIVITY_KIND_COUNT) { + std::lock_guard lock(activityMtx_); + if (msptiActivityDisable(kind) == MSPTI_SUCCESS) { + enabledActivities_.erase(kind); + } else { + LOG(ERROR) << "MsptiMonitor disableActivity failed, kind: " << static_cast(kind); + } + metric::MetricManager::GetInstance()->EnableKindSwitch_(kind, false); + } +} + +void MsptiMonitor::SetFlushInterval(uint32_t interval) +{ + flushInterval_.store(interval); + checkFlush_.store(true); + if (start_.load()) { + cv_.notify_one(); + } + metric::MetricManager::GetInstance()->SetReportInterval(interval); +} + +bool MsptiMonitor::IsStarted() +{ + return start_.load(); +} + +std::set MsptiMonitor::GetEnabledActivities() +{ + std::lock_guard lock(activityMtx_); + return enabledActivities_; +} + +void MsptiMonitor::Run() +{ + if (msptiSubscribe(&subscriber_, nullptr, nullptr) != MSPTI_SUCCESS) { + LOG(ERROR) << "MsptiMonitor run failed, msptiSubscribe failed"; + return; + } + if (msptiActivityRegisterCallbacks(BufferRequest, BufferComplete) != MSPTI_SUCCESS) { + LOG(ERROR) << "MsptiMonitor run failed, msptiActivityRegisterCallbacks failed"; + return; + } + while (true) + { + std::unique_lock lock(cvMtx_); + if (flushInterval_.load() > 0) { + cv_.wait_for(lock, std::chrono::seconds(flushInterval_.load()), + [&]() { return checkFlush_.load() || !start_.load();}); + } else { + cv_.wait(lock, [&]() { return checkFlush_.load () || !start_.load();}); + } + if (!start_.load()) { + break; + } + if (checkFlush_.load()) { + checkFlush_.store(false); + } + if (flushInterval_.load() > 0) { + if (msptiActivityFlushAll(1) != MSPTI_SUCCESS) { + LOG(ERROR) << "MsptiMonitor run msptiActivityFlushAll failed"; + } + } + } + if (msptiUnsubscribe(subscriber_) != MSPTI_SUCCESS) { + LOG(ERROR) << "MsptiMonitor run failed, msptiUnsubscribe failed"; + } + { + std::lock_guard lock(activityMtx_); + for (auto kind : enabledActivities_) { + msptiActivityDisable(kind); + } + enabledActivities_.clear(); + } + checkFlush_.store(false); + flushInterval_.store(0); +} + +std::atomic MsptiMonitor::allocCnt{0}; + +void MsptiMonitor::BufferRequest(uint8_t **buffer, size_t *size, size_t *maxNumRecords) +{ + if (buffer == nullptr || size == nullptr || maxNumRecords == nullptr) { + return; + } + *maxNumRecords = 0; + if (allocCnt.load() >= MAX_ALLOC_CNT) { + *buffer = nullptr; + *size = 0; + LOG(ERROR) << "MsptiMonitor BufferRequest failed, allocCnt: " << allocCnt.load(); + return; + } + uint8_t *pBuffer = ReinterpretConvert(MsptiMalloc(DEFAULT_BUFFER_SIZE, ALIGN_SIZE)); + if (pBuffer == nullptr) { + *buffer = nullptr; + *size = 0; + } else { + *buffer = pBuffer; + *size = DEFAULT_BUFFER_SIZE; + allocCnt++; + LOG(INFO) << "MsptiMonitor BufferRequest, size: " << *size; + } +} + +void MsptiMonitor::BufferComplete(uint8_t *buffer, size_t size, size_t validSize) +{ + if (validSize > 0 && buffer != nullptr) { + LOG(INFO) << "MsptiMonitor BufferComplete, size: " << size << ", validSize: " << validSize; + msptiActivity *record = nullptr; + msptiResult status = MSPTI_SUCCESS; + do { + status = msptiActivityGetNextRecord(buffer, validSize, &record); + if (status == MSPTI_SUCCESS) { + BufferConsume(record); + } else if (status == MSPTI_ERROR_MAX_LIMIT_REACHED) { + break; + } else { + LOG(ERROR) << "MsptiMonitor BufferComplete failed, status: " << static_cast(status); + break; + } + } while (true); + allocCnt--; + } + MsptiFree(buffer); +} + +void MsptiMonitor::BufferConsume(msptiActivity *record) +{ + if (record == nullptr) { + return; + } + metric::MetricManager::GetInstance()->ConsumeMsptiData(record); +} +} // namespace ipc_monitor +} // namespace dynolog_npu diff --git a/msmonitor/plugin/ipc_monitor/mspti_monitor/MsptiMonitor.h b/msmonitor/plugin/ipc_monitor/mspti_monitor/MsptiMonitor.h new file mode 100644 index 0000000000000000000000000000000000000000..f459703fbf7b5027604d7afeac2b3653b8886089 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/mspti_monitor/MsptiMonitor.h @@ -0,0 +1,48 @@ +#ifndef MSPTI_MONITOR_H +#define MSPTI_MONITOR_H + +#include +#include +#include +#include +#include "mspti.h" +#include "thread.h" + + +namespace dynolog_npu { +namespace ipc_monitor { +class MsptiMonitor : public Thread { +public: + explicit MsptiMonitor(); + virtual ~MsptiMonitor(); + void Start(); + void Stop(); + void EnableActivity(msptiActivityKind kind); + void DisableActivity(msptiActivityKind kind); + void SetFlushInterval(uint32_t interval); + bool IsStarted(); + std::set GetEnabledActivities(); + void Uninit(); + +private: + static void BufferRequest(uint8_t **buffer, size_t *size, size_t *maxNumRecords); + static void BufferComplete(uint8_t *buffer, size_t size, size_t validSize); + static void BufferConsume(msptiActivity *record); + static std::atomic allocCnt; + +private: + void Run() override; + +private: + std::atomic start_; + std::mutex cvMtx_; + std::condition_variable cv_; + msptiSubscriberHandle subscriber_; + std::mutex activityMtx_; + std::set enabledActivities_; + std::atomic checkFlush_; + std::atomic flushInterval_; +}; +} // namespace ipc_monitor +} // namespace dynolog_npu +#endif // MSPTI_MONITOR_H diff --git a/msmonitor/plugin/ipc_monitor/mspti_monitor/mspti.h b/msmonitor/plugin/ipc_monitor/mspti_monitor/mspti.h new file mode 100644 index 0000000000000000000000000000000000000000..225dc3b9cb99a8ab1a5cf87322923a27107a0318 --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/mspti_monitor/mspti.h @@ -0,0 +1,244 @@ +#ifndef MSPTI_STUB_H +#define MSPTI_STUB_H + +constexpr int ACTIVITY_STRUCT_ALIGNMENT = 8; +#if defined(_WIN32) +#define START_PACKED_ALIGNMENT __pragma(pack(push, 1)) +#define PACKED_ALIGNMENT __declspec(align(ACTIVITY_STRUCT_ALIGNMENT)) +#define END_PACKED_ALIGNMENT __pragma(pack(pop)) +#elif defined(__GNUC__) +#define START_PACKED_ALIGNMENT +#define PACKED_ALIGNMENT __attribute__((__packed__)) __attribute__((aligned(ACTIVITY_STRUCT_ALIGNMENT))) +#define END_PACKED_ALIGNMENT +#else +#define START_PACKED_ALIGNMENT +#define PACKED_ALIGNMENT +#define END_PACKED_ALIGNMENT +#endif + +#include +#include + +#define MSPTI_INVALID_DEVICE_ID ((uint32_t) 0xFFFFFFFFU) +#define MSPTI_INVALID_STREAM_ID ((uint32_t) 0xFFFFFFFFU) +#define MSPTI_INVALID_CORRELATION_ID ((uint64_t) 0) +using msptiCallbackId = uint32_t; + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +typedef enum { + MSPTI_SUCCESS = 0, + MSPTI_ERROR_INVALID_PARAMETER = 1, + MSPTI_ERROR_MULTIPLE_SUBSCRIBERS_NOT_SUPPORTED = 2, + MSPTI_ERROR_MAX_LIMIT_REACHED = 3, + MSPTI_ERROR_DEVICE_OFFLINE = 4, + MSPTI_ERROR_QUERY_EMPTY = 5, + MSPTI_ERROR_INNER = 999, + MSPTI_ERROR_FOECE_INT = 0x7fffffff +} msptiResult; + +typedef enum { + MSPTI_CB_DOMAIN_INVALID = 0, + MSPTI_CB_DOMAIN_RUNTIME = 1, + MSPTI_CB_DOMAIN_HCCL = 2, + MSPTI_CB_DOMAIN_SIZE, + MSPTI_CB_DOMAIN_FORCE_INT = 0x7fffffff +} msptiCallbackDomain; + +typedef enum { + MSPTI_API_ENTER = 0, + MSPTI_API_EXIT = 1, + MSPTI_API_CBSITE_FORCE_INT = 0x7fffffff +} msptiApiCallbackSite; + +typedef struct { + msptiApiCallbackSite callbackSite; + const char *functionName; + const void *functionParams; + const void *functionReturnValue; + const char *symbolName; + uint64_t correlationId; + uint64_t reserved1; + uint64_t reserved2; + uint64_t *correlationData; +} msptiCallbackData; + +typedef enum { + MSPTI_ACTIVITY_KIND_INVALID = 0, + MSPTI_ACTIVITY_KIND_MARKER = 1, + MSPTI_ACTIVITY_KIND_KERNEL = 2, + MSPTI_ACTIVITY_KIND_API = 3, + MSPTI_ACTIVITY_KIND_HCCL = 4, + MSPTI_ACTIVITY_KIND_MEMORY = 5, + MSPTI_ACTIVITY_KIND_MEMSET = 6, + MSPTI_ACTIVITY_KIND_MEMCPY = 7, + MSPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION = 8, + MSPTI_ACTIVITY_KIND_COUNT, + MSPTI_ACTIVITY_KIND_FORCE_INT = 0x7fffffff +} msptiActivityKind; + +typedef enum { + MSPTI_ACTIVITY_FLAG_NONE = 0, + MSPTI_ACTIVITY_FLAG_MARKER_INSTANTANEOUS = 1 << 0, + MSPTI_ACTIVITY_FLAG_MARKER_START = 1 << 1, + MSPTI_ACTIVITY_FLAG_MARKER_END = 1 << 2, + MSPTI_ACTIVITY_FLAG_MARKER_INSTANTANEOUS_WITH_DEVICE = 1 << 3, + MSPTI_ACTIVITY_FLAG_MARKER_START_WITH_DEVICE = 1 << 4, + MSPTI_ACTIVITY_FLAG_MARKER_END_WITH_DEVICE = 1 << 5 +} msptiActivityFlag; + +typedef enum { + MSPTI_ACTIVITY_SOURCE_KIND_HOST = 0, + MSPTI_ACTIVITY_SOURCE_KIND_DEVICE = 1 +} msptiActivitySourceKind; + +typedef enum { + MSPTI_ACTIVITY_MEMORY_OPERATION_TYPE_ALLOCATATION = 0, + MSPTI_ACTIVITY_MEMORY_OPERATION_TYPE_RELEASE = 1 +} msptiActivityMemoryOperationType; + +typedef enum { + MSPTI_ACTIVITY_MEMORY_KIND_UNKNOWN = 0, + MSPTI_ACTIVITY_MEMORY_KIND_DEVICE = 1 +} msptiActivityMemoryKind; + +typedef enum { + MSPTI_ACTIVITY_MEMCPY_KIND_UNKNOWN = 0, + MSPTI_ACTIVITY_MEMCPY_KIND_HTOH = 1, + MSPTI_ACTIVITY_MEMCPY_KIND_HTOD = 2, + MSPTI_ACTIVITY_MEMCPY_KIND_DTOH = 3, + MSPTI_ACTIVITY_MEMCPY_KIND_DTOD = 4, + MSPTI_ACTIVITY_MEMCPY_KIND_DEFAULT = 5 +} msptiActivityMemcpyKind; + +START_PACKED_ALIGNMENT + +typedef union PACKED_ALIGNMENT { + struct { + uint32_t processId; + uint32_t threadId; + } pt; + struct { + uint32_t deviceId; + uint32_t streamId; + } ds; +} msptiObjectId; + +typedef struct PACKED_ALIGNMENT { + msptiActivityKind kind; +} msptiActivity; + +typedef struct PACKED_ALIGNMENT { + msptiActivityKind kind; + uint64_t start; + uint64_t end; + struct { + uint32_t processId; + uint32_t threadId; + } pt; + uint64_t correlationId; + const char* name; +} msptiActivityApi; + +typedef struct PACKED_ALIGNMENT { + msptiActivityKind kind; + uint64_t start; + uint64_t end; + struct { + uint32_t deviceId; + uint32_t streamId; + } ds; + uint64_t correlationId; + const char *type; + const char *name; +} msptiActivityKernel; + +typedef struct PACKED_ALIGNMENT { + msptiActivityKind kind; + msptiActivityFlag flag; + msptiActivitySourceKind sourceKind; + uint64_t timestamp; + uint64_t id; + msptiObjectId objectId; + const char *name; + const char *domain; +} msptiActivityMarker; + +typedef struct PACKED_ALIGNMENT { + msptiActivityKind kind; + uint64_t start; + uint64_t end; + struct { + uint32_t deviceId; + uint32_t streamId; + } ds; + double bandWidth; + const char *name; + const char *commName; +} msptiActivityHccl; + +typedef struct PACKED_ALIGNMENT { + msptiActivityKind kind; + msptiActivityMemoryOperationType memoryOperationType; + msptiActivityMemoryKind memoryKind; + uint64_t correlationId; + uint64_t start; + uint64_t end; + uint64_t address; + uint64_t bytes; + uint32_t processId; + uint32_t deviceId; + uint32_t streamId; +} msptiActivityMemory; + +typedef struct PACKED_ALIGNMENT { + msptiActivityKind kind; + uint32_t value; + uint64_t bytes; + uint64_t start; + uint64_t end; + uint32_t deviceId; + uint32_t streamId; + uint64_t correlationId; + uint8_t isAsync; +} msptiActivityMemset; + +typedef struct PACKED_ALIGNMENT { + msptiActivityKind kind; + msptiActivityMemcpyKind copyKind; + uint64_t bytes; + uint64_t start; + uint64_t end; + uint32_t deviceId; + uint32_t streamId; + uint64_t correlationId; + uint8_t isAsync; +} msptiActivityMemcpy; + +END_PACKED_ALIGNMENT + +typedef void(*msptiCallbackFunc)(void* userdata, msptiCallbackDomain domain, msptiCallbackId cbid, const msptiCallbackData *cbdata); +typedef void(*msptiBuffersCallbackRequestFunc)(uint8_t **buffer, size_t *size, size_t *maxNumRecords); +typedef void(*msptiBuffersCallbackCompleteFunc)(uint8_t *buffer, size_t size, size_t validSize); + +struct msptiSubscriber_st { + msptiCallbackFunc callback; + void *userdata; +}; + +typedef struct msptiSubscriber_st *msptiSubscriberHandle; + +msptiResult msptiSubscribe(msptiSubscriberHandle *subscriber, msptiCallbackFunc callback, void *userdata); +msptiResult msptiUnsubscribe(msptiSubscriberHandle subscriber); +msptiResult msptiActivityRegisterCallbacks(msptiBuffersCallbackRequestFunc funcBufferRequested, msptiBuffersCallbackCompleteFunc funcBufferCompleted); +msptiResult msptiActivityEnable(msptiActivityKind kind); +msptiResult msptiActivityDisable(msptiActivityKind kind); +msptiResult msptiActivityGetNextRecord(uint8_t *buffer, size_t validBufferSizeBytes, msptiActivity **record); +msptiResult msptiActivityFlushAll(uint32_t flag); + +#ifdef __cplusplus +} +#endif // __cplusplus +#endif // MSPTI_STUB_H diff --git a/dynolog_npu/plugin/ipc_monitor/singleton.h b/msmonitor/plugin/ipc_monitor/singleton.h similarity index 95% rename from dynolog_npu/plugin/ipc_monitor/singleton.h rename to msmonitor/plugin/ipc_monitor/singleton.h index 8bb106f3adc8b365ef81feb603c6aaac917a00e2..b2e874dc04f4720571ea178047e34b23641ae08c 100644 --- a/dynolog_npu/plugin/ipc_monitor/singleton.h +++ b/msmonitor/plugin/ipc_monitor/singleton.h @@ -1,31 +1,31 @@ -#ifndef SINGLETON_H -#define SINGLETON_H -#include - -namespace dynolog_npu { -namespace ipc_monitor { - -template -class Singleton { -public: - static T *GetInstance() noexcept(std::is_nothrow_constructible::value) { - static T instance; - return &instance; - } - - virtual ~Singleton() = default; - -protected: - explicit Singleton() = default; - -private: - explicit Singleton(const Singleton &obj) = delete; - Singleton& operator=(const Singleton &obj) = delete; - explicit Singleton(Singleton &&obj) = delete; - Singleton& operator=(Singleton &&obj) = delete; -}; - -} // ipc_monitor -} // dynolog_npu - +#ifndef SINGLETON_H +#define SINGLETON_H +#include + +namespace dynolog_npu { +namespace ipc_monitor { + +template +class Singleton { +public: + static T *GetInstance() noexcept(std::is_nothrow_constructible::value) { + static T instance; + return &instance; + } + + virtual ~Singleton() = default; + +protected: + explicit Singleton() = default; + +private: + explicit Singleton(const Singleton &obj) = delete; + Singleton& operator=(const Singleton &obj) = delete; + explicit Singleton(Singleton &&obj) = delete; + Singleton& operator=(Singleton &&obj) = delete; +}; + +} // ipc_monitor +} // dynolog_npu + #endif \ No newline at end of file diff --git a/msmonitor/plugin/ipc_monitor/thread.h b/msmonitor/plugin/ipc_monitor/thread.h new file mode 100644 index 0000000000000000000000000000000000000000..9e1926917af380ec14cb517cb9efe57bf110405a --- /dev/null +++ b/msmonitor/plugin/ipc_monitor/thread.h @@ -0,0 +1,75 @@ +#ifndef IPC_MONITOR_THREAD_H +#define IPC_MONITOR_THREAD_H + +#include +#include +#include +#include +#include "utils.h" + +namespace dynolog_npu { +namespace ipc_monitor { +class Thread { +public: + Thread() + : is_alive_(false), + pid_(0), + thread_name_("IPCMonitor") {} + + ~Thread() + { + if (is_alive_) { + (void)pthread_cancel(pid_); + (void)pthread_join(pid_, nullptr); + } + } + + void SetThreadName(const std::string &name) + { + if (!name.empty()) { + thread_name_ = name; + } + } + + std::string GetThreadName() + { + return thread_name_; + } + + int Start() + { + int ret = pthread_create(&pid_, nullptr, Execute, ReinterpretConvert(this)); + is_alive_ = (ret == 0) ? true : false; + return ret; + } + + int Stop() + { + return Join(); + } + + int Join() + { + int ret = pthread_join(pid_, nullptr); + is_alive_ = (ret == 0) ? false : true; + return ret; + } + +private: + static void* Execute(void *args) + { + Thread *thr = ReinterpretConvert(args); + prctl(PR_SET_NAME, ReinterpretConvert(thr->GetThreadName().data())); + thr->Run(); + return nullptr; + } + virtual void Run() = 0; + +private: + bool is_alive_; + pthread_t pid_; + std::string thread_name_; +}; +} // ipc_monitor +} // dynolog_npu +#endif // IPC_MONITOR_THREAD_H diff --git a/dynolog_npu/plugin/ipc_monitor/utils.cpp b/msmonitor/plugin/ipc_monitor/utils.cpp similarity index 33% rename from dynolog_npu/plugin/ipc_monitor/utils.cpp rename to msmonitor/plugin/ipc_monitor/utils.cpp index b57942082e0fd52426ddce47bfc70620bf19019f..ac9ac236c2f8cd62b04a246215e930449152ab7c 100644 --- a/dynolog_npu/plugin/ipc_monitor/utils.cpp +++ b/msmonitor/plugin/ipc_monitor/utils.cpp @@ -1,4 +1,20 @@ #include "utils.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include namespace dynolog_npu { namespace ipc_monitor { @@ -38,6 +54,14 @@ std::string getCurrentTimestamp() return oss.str(); } +uint64_t getCurrentTimestamp64() +{ + auto now = std::chrono::system_clock::now(); + auto micros = std::chrono::duration_cast(now.time_since_epoch()); + auto milli_time = std::chrono::duration_cast(micros).count(); + return milli_time; +} + std::string formatErrorCode(SubModule submodule, ErrCode errorCode) { std::ostringstream oss; @@ -45,16 +69,36 @@ std::string formatErrorCode(SubModule submodule, ErrCode errorCode) oss << "ERR" << std::setw(2) << std::setfill('0') << static_cast(submodule); // 2: 字段宽度 oss << std::setw(3) << std::setfill('0') << static_cast(errorCode); // 3: 字段宽度 oss << " " << submoduleMap[submodule] << " " << errCodeMap[errorCode]; - return oss.str(); }; - int32_t GetProcessId() { return static_cast(getpid()); } +bool ParseProcStat(const std::string& line, std::string& command, int& parentPid) +{ + size_t lparen = line.find('('); + size_t rparen = line.rfind(')'); + if (lparen == std::string::npos || rparen == std::string::npos || rparen <= lparen + 1) { + LOG(WARNING) << "cannot find command name: " << line; + return false; + } + command = line.substr(lparen + 1, rparen - lparen - 1); + + std::string afterCmd = line.substr(rparen + 1); + std::istringstream iss(afterCmd); + std::string state; + int ppid; + if (!(iss >> state >> ppid)) { + LOG(WARNING) << "Failed to parse state/ppid from: " << afterCmd; + return false; + } + parentPid = ppid; + return true; +} + std::pair GetParentPidAndCommand(int32_t pid) { std::string fileName = "/proc/" + std::to_string(pid) + "/stat"; @@ -66,13 +110,12 @@ std::pair GetParentPidAndCommand(int32_t pid) std::string command; std::string line; if (std::getline(statFile, line)) { - int ret = sscanf(line.c_str(), "%*d (%[^)]) %*c %d", command.data(), &parentPid); - if (ret == 2) { // 2: 接收到2个字符 - LOG(INFO) << "Success to get parent pid: " << parentPid; + bool ret = ParseProcStat(line, command, parentPid); + if (ret) { return std::make_pair(parentPid, command); } } - LOG(ERROR) << " Failed to parse /proc/" << pid << "/stat"; + LOG(WARNING) << "Failed to parse /proc/" << pid << "/stat"; return std::make_pair(0, ""); } @@ -97,8 +140,10 @@ std::vector GetPids() for (const auto &pidPair : pids) { res.push_back(pidPair.first); } + LOG(INFO) << "Success to get parent pid: " << res; return res; } + std::string GenerateUuidV4() { static std::random_device randomDevice; @@ -131,5 +176,258 @@ std::string GenerateUuidV4() return stringStream.str(); } +bool Str2Uint32(uint32_t& dest, const std::string& str) +{ + if (str.empty()) { + LOG(ERROR) << "Str to uint32 failed, input string is null"; + return false; + } + size_t pos = 0; + try { + dest = static_cast(std::stoul(str, &pos)); + } catch(...) { + LOG(ERROR) << "Str to uint32 failed, input string is " << str; + return false; + } + if (pos != str.size()) { + LOG(ERROR) << "Str to uint32 failed, input string is " << str; + return false; + } + return true; +} + +bool Str2Bool(bool& dest, const std::string& str) +{ + std::string lower_str = str; + std::transform(lower_str.begin(), lower_str.end(), lower_str.begin(), ::tolower); + + if (lower_str == "true" || lower_str == "1") { + dest = true; + return true; + } + + if (lower_str == "false" || lower_str == "0") { + dest = false; + return true; + } + LOG(ERROR) << "Str to bool failed, input string is " << str; + return false; +} + +std::string& trim(std::string& str) +{ + if (str.empty()) { + return str; + } + str.erase(0, str.find_first_not_of(" ")); + str.erase(str.find_last_not_of(" ") + 1); + return str; +} + +// split函数 +std::vector split(const std::string& str, char delimiter) +{ + std::vector tokens; + std::string token; + std::istringstream tokenStream(str); + + while (std::getline(tokenStream, token, delimiter)) { + tokens.push_back(token); + } + + return tokens; +} + +void *MsptiMalloc(size_t size, size_t alignment) +{ + if (alignment > 0) { + size = (size + alignment - 1) / alignment * alignment; + } +#if defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200112L + void *ptr = nullptr; + if (posix_memalign(&ptr, alignment, size) != 0) { + ptr = nullptr; + } + return ptr; +#else + return malloc(size); +#endif +} + +void MsptiFree(uint8_t *ptr) +{ + if (ptr != nullptr) { + free(ptr); + } +} + +bool PathUtils::IsFileExist(const std::string &path) +{ + if (path.empty() || path.size() > PATH_MAX) { + return false; + } + return access(path.c_str(), F_OK) == 0; +} + +bool PathUtils::IsFileWritable(const std::string &path) +{ + if (path.empty() || path.size() > PATH_MAX) { + return false; + } + return access(path.c_str(), W_OK) == 0; +} + +bool PathUtils::IsDir(const std::string &path) +{ + if (path.empty() || path.size() > PATH_MAX) { + return false; + } + struct stat st{}; + int ret = lstat(path.c_str(), &st); + if (ret != 0) { + return false; + } + return S_ISDIR(st.st_mode); +} + +bool PathUtils::CreateDir(const std::string &path) +{ + if (path.empty() || path.size() > PATH_MAX) { + return false; + } + if (IsFileExist(path)) { + return IsDir(path); + } + size_t pos = 0; + while ((pos = path.find_first_of('/', pos)) != std::string::npos) { + std::string baseDir = path.substr(0, ++pos); + if (IsFileExist(baseDir)) { + if (IsDir(baseDir)) { + continue; + } else { + return false; + } + } + if (mkdir(baseDir.c_str(), DATA_DIR_AUTHORITY) != 0) { + if (errno != EEXIST) { + return false; + } + } + } + auto ret = mkdir(path.c_str(), DATA_DIR_AUTHORITY); + return (ret == 0 || errno == EEXIST) ? true : false; +} + +std::string PathUtils::RealPath(const std::string &path) +{ + if (path.empty() || path.size() > PATH_MAX) { + return ""; + } + char realPath[PATH_MAX] = {0}; + if (realpath(path.c_str(), realPath) == nullptr) { + return ""; + } + return std::string(realPath); +} + +std::string PathUtils::RelativeToAbsPath(const std::string &path) +{ + if (path.empty() || path.size() > PATH_MAX) { + return ""; + } + if (path[0] != '/') { + char pwdPath[PATH_MAX] = {0}; + if (getcwd(pwdPath, PATH_MAX) != nullptr) { + return std::string(pwdPath) + "/" + path; + } + return ""; + } + return std::string(path); +} + +std::string PathUtils::DirName(const std::string &path) +{ + if (path.empty()) { + return ""; + } + char tempPath[PATH_MAX] = {0}; + strncpy(tempPath, path.c_str(), path.size() < PATH_MAX ? path.size() : PATH_MAX); + char* cPath = dirname(tempPath); + return cPath ? std::string(cPath) : ""; +} + +bool PathUtils::CreateFile(const std::string &path) +{ + if (path.empty() || path.size() > PATH_MAX || !CreateDir(DirName(path))) { + return false; + } + int fd = creat(path.c_str(), DATA_FILE_AUTHORITY); + return (fd < 0 || close(fd) != 0) ? false : true; +} + +bool PathUtils::IsSoftLink(const std::string &path) +{ + if (path.empty() || path.size() > PATH_MAX || !IsFileExist(path)) { + return false; + } + struct stat st{}; + if (lstat(path.c_str(), &st) != 0) { + return false; + } + return S_ISLNK(st.st_mode); +} + +bool PathUtils::DirPathCheck(const std::string& absPath) +{ + if (absPath.empty() || absPath.size() > PATH_MAX) { + fprintf(stderr, "[ERROR] The length of Path %s is invalid.\n", absPath.c_str()); + return false; + } + if (IsSoftLink(absPath)) { + fprintf(stderr, "[ERROR] Path %s is soft link.\n", absPath.c_str()); + return false; + } + if (!IsFileExist(absPath) && !CreateDir(absPath)) { + fprintf(stderr, "[ERROR] Path %s not exist and create failed.\n", absPath.c_str()); + return false; + } + if (!IsDir(absPath) || !IsFileWritable(absPath)) { + fprintf(stderr, "[ERROR] %s is not a directory or is not writable.\n", absPath.c_str()); + return false; + } + return true; +} + +bool CreateMsmonitorLogPath(std::string& path) +{ + const char* logPathEnvVal = getenv("MSMONITOR_LOG_PATH"); + std::string logPath; + if (logPathEnvVal != nullptr) { + logPath = logPathEnvVal; + } + if (logPath.empty()) { + char cwdPath[PATH_MAX] = {0}; + if (getcwd(cwdPath, PATH_MAX) != nullptr) { + logPath = cwdPath; + } + } + if (logPath.empty()) { + fprintf(stderr, "[ERROR] Failed to get msmonitor log path.\n"); + return false; + } + logPath = logPath + "/msmonitor_log"; + std::string absPath = PathUtils::RelativeToAbsPath(logPath); + if (PathUtils::DirPathCheck(absPath)) { + std::string realPath = PathUtils::RealPath(absPath); + if (PathUtils::CreateDir(realPath)) { + path = realPath; + return true; + } + fprintf(stderr, "[ERROR] Create LOG_PATH: %s failed.\n", realPath.c_str()); + } else { + fprintf(stderr, "[ERROR] LOG_PATH: %s of Msmonitor is invalid.\n", absPath.c_str()); + } + return false; +} } // namespace ipc_monitor } // namespace dynolog_npu diff --git a/dynolog_npu/plugin/ipc_monitor/utils.h b/msmonitor/plugin/ipc_monitor/utils.h similarity index 33% rename from dynolog_npu/plugin/ipc_monitor/utils.h rename to msmonitor/plugin/ipc_monitor/utils.h index 2374a27d417f91bc23108a892c6eb25cbb5039d8..df452ea4d2d56bbaa9e7a74bc69bae11f74ce2f4 100644 --- a/dynolog_npu/plugin/ipc_monitor/utils.h +++ b/msmonitor/plugin/ipc_monitor/utils.h @@ -1,22 +1,31 @@ +/* + * Copyright (C) 2025-2025. Huawei Technologies Co., Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #ifndef IPC_MONITOR_UTILS_H #define IPC_MONITOR_UTILS_H -#include -#include + #include #include #include -#include -#include -#include -#include -#include -#include +#include #include - +#include namespace dynolog_npu { namespace ipc_monitor { - constexpr int MaxParentPids = 5; int32_t GetProcessId(); std::string GenerateUuidV4(); @@ -24,6 +33,18 @@ std::vector GetPids(); std::pair GetParentPidAndCommand(int32_t pid); std::vector> GetPidCommandPairsofAncestors(); std::string getCurrentTimestamp(); +uint64_t getCurrentTimestamp64(); +bool Str2Uint32(uint32_t& dest, const std::string& str); +bool Str2Bool(bool& dest, const std::string& str); +std::string& trim(std::string& str); +std::vector split(const std::string& str, char delimiter); + +constexpr size_t ALIGN_SIZE = 8; +void *MsptiMalloc(size_t size, size_t alignment); +void MsptiFree(uint8_t *ptr); +const mode_t DATA_FILE_AUTHORITY = 0640; +const mode_t DATA_DIR_AUTHORITY = 0750; +const int DEFAULT_FLUSH_INTERVAL = 60; enum class SubModule { IPC = 0 @@ -45,7 +66,6 @@ enum class ErrCode { PERMISSION = 12, }; - std::string formatErrorCode(SubModule submodule, ErrCode errorCode); #define IPC_ERROR(error) formatErrorCode(SubModule::IPC, error) @@ -54,10 +74,31 @@ template inline T ReinterpretConvert(V ptr) { return reinterpret_cast(ptr); } +template +auto groupby(const Container& vec, KeyFunc keyFunc) { + using KeyType = decltype(keyFunc(*vec.begin())); + using ValueType = typename Container::value_type; + std::unordered_map> grouped; + for (const auto& item : vec) { + grouped[keyFunc(item)].push_back(item); + } + return grouped; +} +bool CreateMsmonitorLogPath(std::string& path); +struct PathUtils { + static bool IsFileExist(const std::string &path); + static bool IsFileWritable(const std::string &path); + static bool IsDir(const std::string &path); + static bool CreateDir(const std::string &path); + static std::string RealPath(const std::string &path); + static std::string RelativeToAbsPath(const std::string &path); + static std::string DirName(const std::string &path); + static bool CreateFile(const std::string &path); + static bool IsSoftLink(const std::string &path); + static bool DirPathCheck(const std::string &path); +}; } // namespace ipc_monitor } // namespace dynolog_npu - -#endif - +#endif // IPC_MONITOR_UTILS_H diff --git a/msmonitor/plugin/setup.py b/msmonitor/plugin/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..2e257a48ada719a56d3cd0299f56f61351f249f4 --- /dev/null +++ b/msmonitor/plugin/setup.py @@ -0,0 +1,69 @@ +# Copyright (c) 2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import sys + +import subprocess +import pybind11 + +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext + + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=""): + super().__init__(name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + + +class CMakeBuild(build_ext): + def run(self): + for ext in self.extensions: + self.build_extension(ext) + + def build_extension(self, ext): + cfg = 'Debug' if self.debug else 'Release' + build_args = ['--config', cfg] + + ext_dir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + cmake_args = [ + '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + ext_dir, + '-DPYTHON_EXECUTABLE=' + sys.executable, + '-DCMAKE_PREFIX_PATH=' + pybind11.get_cmake_dir(), + '-DCMAKE_INSTALL_PREFIX=' + ext_dir, + '-DDYNOLOG_PATH=' + os.path.join(os.path.dirname(BASE_DIR), "third_party", "dynolog"), + '-DCMAKE_BUILD_TYPE=' + cfg + ] + + env = os.environ.copy() + env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''), + self.distribution.get_version()) + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env) + subprocess.check_call(['cmake', '--build', '.', '--target', 'install', '-j', '8'] + build_args, + cwd=self.build_temp) + +BASE_DIR = os.path.dirname(os.path.realpath(__file__)) + +setup( + name="msmonitor_plugin", + version="0.1", + description="msMonitor plugins", + ext_modules=[CMakeExtension('IPCMonitor')], + cmdclass=dict(build_ext=CMakeBuild), + install_requires=["pybind11"], +) diff --git a/msmonitor/plugin/stub/build_stub.sh b/msmonitor/plugin/stub/build_stub.sh new file mode 100644 index 0000000000000000000000000000000000000000..97ec0699aec5923497ee32a7252b0337db059f7f --- /dev/null +++ b/msmonitor/plugin/stub/build_stub.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +CDIR="$(cd "$(dirname "$0")" ; pwd -P)" + +cd ${CDIR} + +gcc -fPIC -shared -o libmspti.so -I../ipc_monitor/mspti_monitor mspti.cpp diff --git a/msmonitor/plugin/stub/mspti.cpp b/msmonitor/plugin/stub/mspti.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d0c73b74430b9d33bff35e9e0f0a2912bda0b354 --- /dev/null +++ b/msmonitor/plugin/stub/mspti.cpp @@ -0,0 +1,36 @@ +#include "mspti.h" + +msptiResult msptiSubscribe(msptiSubscriberHandle *subscriber, msptiCallbackFunc callback, void *userdata) +{ + return MSPTI_SUCCESS; +} + +msptiResult msptiUnsubscribe(msptiSubscriberHandle subscriber) +{ + return MSPTI_SUCCESS; +} + +msptiResult msptiActivityRegisterCallbacks(msptiBuffersCallbackRequestFunc funcBufferRequested, msptiBuffersCallbackCompleteFunc funcBufferCompleted) +{ + return MSPTI_SUCCESS; +} + +msptiResult msptiActivityEnable(msptiActivityKind kind) +{ + return MSPTI_SUCCESS; +} + +msptiResult msptiActivityDisable(msptiActivityKind kind) +{ + return MSPTI_SUCCESS; +} + +msptiResult msptiActivityGetNextRecord(uint8_t *buffer, size_t validBufferSizeBytes, msptiActivity **record) +{ + return MSPTI_SUCCESS; +} + +msptiResult msptiActivityFlushAll(uint32_t flag) +{ + return MSPTI_SUCCESS; +} diff --git a/msmonitor/plugin/third_party/securec/include/securec.h b/msmonitor/plugin/third_party/securec/include/securec.h new file mode 100644 index 0000000000000000000000000000000000000000..fa575ffe359104deabd5d32154c9afbc81065ddf --- /dev/null +++ b/msmonitor/plugin/third_party/securec/include/securec.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2014-2021. All rights reserved. + * Licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * Description: The user of this secure c library should include this header file in you source code. + * This header file declare all supported API prototype of the library, + * such as memcpy_s, strcpy_s, wcscpy_s,strcat_s, strncat_s, sprintf_s, scanf_s, and so on. + * Create: 2014-02-25 + * Notes: Do not modify this file by yourself. + */ + +#ifndef SECUREC_H_5D13A042_DC3F_4ED9_A8D1_882811274C27 +#define SECUREC_H_5D13A042_DC3F_4ED9_A8D1_882811274C27 + +#include "securectype.h" +#ifndef SECUREC_HAVE_STDARG_H +#define SECUREC_HAVE_STDARG_H 1 +#endif + +#if SECUREC_HAVE_STDARG_H +#include +#endif + +#ifndef SECUREC_HAVE_ERRNO_H +#define SECUREC_HAVE_ERRNO_H 1 +#endif + +/* EINVAL ERANGE may defined in errno.h */ +#if SECUREC_HAVE_ERRNO_H +#if SECUREC_IN_KERNEL +#include +#else +#include +#endif +#endif + +/* Define error code */ +#if defined(SECUREC_NEED_ERRNO_TYPE) || !defined(__STDC_WANT_LIB_EXT1__) || \ + (defined(__STDC_WANT_LIB_EXT1__) && (!__STDC_WANT_LIB_EXT1__)) +#ifndef SECUREC_DEFINED_ERRNO_TYPE +#define SECUREC_DEFINED_ERRNO_TYPE +/* Just check whether macrodefinition exists. */ +#ifndef errno_t +typedef int errno_t; +#endif +#endif +#endif + +/* Success */ +#ifndef EOK +#define EOK 0 +#endif + +#ifndef EINVAL +/* The src buffer is not correct and destination buffer can not be reset */ +#define EINVAL 22 +#endif + +#ifndef EINVAL_AND_RESET +/* Once the error is detected, the dest buffer must be reset! Value is 22 or 128 */ +#define EINVAL_AND_RESET 150 +#endif + +#ifndef ERANGE +/* The destination buffer is not long enough and destination buffer can not be reset */ +#define ERANGE 34 +#endif + +#ifndef ERANGE_AND_RESET +/* Once the error is detected, the dest buffer must be reset! Value is 34 or 128 */ +#define ERANGE_AND_RESET 162 +#endif + +#ifndef EOVERLAP_AND_RESET +/* Once the buffer overlap is detected, the dest buffer must be reset! Value is 54 or 128 */ +#define EOVERLAP_AND_RESET 182 +#endif + +/* If you need export the function of this library in Win32 dll, use __declspec(dllexport) */ +#ifndef SECUREC_API +#if defined(SECUREC_DLL_EXPORT) +#if defined(_MSC_VER) +#define SECUREC_API __declspec(dllexport) +#else /* build for linux */ +#define SECUREC_API __attribute__((visibility("default"))) +#endif /* end of _MSC_VER and SECUREC_DLL_EXPORT */ +#elif defined(SECUREC_DLL_IMPORT) +#if defined(_MSC_VER) +#define SECUREC_API __declspec(dllimport) +#else +#define SECUREC_API +#endif /* end of _MSC_VER and SECUREC_DLL_IMPORT */ +#else +/* + * Standardized function declaration. If a security function is declared in the your code, + * it may cause a compilation alarm,Please delete the security function you declared. + * Adding extern under windows will cause the system to have inline functions to expand, + * so do not add the extern in default + */ +#if defined(_MSC_VER) +#define SECUREC_API +#else +#define SECUREC_API extern +#endif +#endif +#endif + +#ifdef __cplusplus +extern "C" { +#endif +/* + * Description: The GetHwSecureCVersion function get SecureC Version string and version number. + * Parameter: verNumber - to store version number (for example value is 0x500 | 0xa) + * Return: version string + */ +SECUREC_API const char *GetHwSecureCVersion(unsigned short *verNumber); + +#if SECUREC_ENABLE_MEMSET +/* + * Description: The memset_s function copies the value of c (converted to an unsigned char) into each of + * the first count characters of the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax - The maximum length of destination buffer + * Parameter: c - the value to be copied + * Parameter: count - copies count bytes of value to dest + * Return: EOK if there was no runtime-constraint violation + */ +SECUREC_API errno_t memset_s(void *dest, size_t destMax, int c, size_t count); +#endif + +#ifndef SECUREC_ONLY_DECLARE_MEMSET +#define SECUREC_ONLY_DECLARE_MEMSET 0 +#endif + +#if !SECUREC_ONLY_DECLARE_MEMSET + +#if SECUREC_ENABLE_MEMCPY +/* + * Description: The memcpy_s function copies n characters from the object pointed to + * by src into the object pointed to by dest. + * Parameter: dest - destination address + * Parameter: destMax - The maximum length of destination buffer + * Parameter: src - source address + * Parameter: count - copies count bytes from the src + * Return: EOK if there was no runtime-constraint violation + */ +SECUREC_API errno_t memcpy_s(void *dest, size_t destMax, const void *src, size_t count); +#endif + +#endif + +#ifdef __cplusplus +} +#endif +#endif diff --git a/msmonitor/plugin/third_party/securec/include/securectype.h b/msmonitor/plugin/third_party/securec/include/securectype.h new file mode 100644 index 0000000000000000000000000000000000000000..c406d198971a926ead3f9564072a6d9e828e6894 --- /dev/null +++ b/msmonitor/plugin/third_party/securec/include/securectype.h @@ -0,0 +1,501 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2014-2021. All rights reserved. + * Licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * Description: Define internal used macro and data type. The marco of SECUREC_ON_64BITS + * will be determined in this header file, which is a switch for part + * of code. Some macro are used to suppress warning by MS compiler. + * Create: 2014-02-25 + * Notes: User can change the value of SECUREC_STRING_MAX_LEN and SECUREC_MEM_MAX_LEN + * macro to meet their special need, but The maximum value should not exceed 2G. + */ +/* + * [Standardize-exceptions]: Performance-sensitive + * [reason]: Strict parameter verification has been done before use + */ + +#ifndef SECURECTYPE_H_A7BBB686_AADA_451B_B9F9_44DACDAE18A7 +#define SECURECTYPE_H_A7BBB686_AADA_451B_B9F9_44DACDAE18A7 + +#ifndef SECUREC_USING_STD_SECURE_LIB +#if defined(_MSC_VER) && _MSC_VER >= 1400 +#if defined(__STDC_WANT_SECURE_LIB__) && (!__STDC_WANT_SECURE_LIB__) +/* Security functions have been provided since vs2005, default use of system library functions */ +#define SECUREC_USING_STD_SECURE_LIB 0 +#else +#define SECUREC_USING_STD_SECURE_LIB 1 +#endif +#else +#define SECUREC_USING_STD_SECURE_LIB 0 +#endif +#endif + +/* Compatibility with older Secure C versions, shielding VC symbol redefinition warning */ +#if defined(_MSC_VER) && (_MSC_VER >= 1400) && (!SECUREC_USING_STD_SECURE_LIB) +#ifndef SECUREC_DISABLE_CRT_FUNC +#define SECUREC_DISABLE_CRT_FUNC 1 +#endif +#ifndef SECUREC_DISABLE_CRT_IMP +#define SECUREC_DISABLE_CRT_IMP 1 +#endif +#else /* MSC VER */ +#ifndef SECUREC_DISABLE_CRT_FUNC +#define SECUREC_DISABLE_CRT_FUNC 0 +#endif +#ifndef SECUREC_DISABLE_CRT_IMP +#define SECUREC_DISABLE_CRT_IMP 0 +#endif +#endif + +#if SECUREC_DISABLE_CRT_FUNC +#ifdef __STDC_WANT_SECURE_LIB__ +#undef __STDC_WANT_SECURE_LIB__ +#endif +#define __STDC_WANT_SECURE_LIB__ 0 +#endif + +#if SECUREC_DISABLE_CRT_IMP +#ifdef _CRTIMP_ALTERNATIVE +#undef _CRTIMP_ALTERNATIVE +#endif +#define _CRTIMP_ALTERNATIVE /* Comment Microsoft *_s function */ +#endif + +/* Compile in kernel under macro control */ +#ifndef SECUREC_IN_KERNEL +#ifdef __KERNEL__ +#define SECUREC_IN_KERNEL 1 +#else +#define SECUREC_IN_KERNEL 0 +#endif +#endif + +/* make kernel symbols of functions available to loadable modules */ +#ifndef SECUREC_EXPORT_KERNEL_SYMBOL +#if SECUREC_IN_KERNEL +#define SECUREC_EXPORT_KERNEL_SYMBOL 1 +#else +#define SECUREC_EXPORT_KERNEL_SYMBOL 0 +#endif +#endif + +#if SECUREC_IN_KERNEL +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 0 +#endif +#ifndef SECUREC_ENABLE_WCHAR_FUNC +#define SECUREC_ENABLE_WCHAR_FUNC 0 +#endif +#else /* SECUREC_IN_KERNEL */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 1 +#endif +#ifndef SECUREC_ENABLE_WCHAR_FUNC +#define SECUREC_ENABLE_WCHAR_FUNC 1 +#endif +#endif + +/* Default secure function declaration, default declarations for non-standard functions */ +#ifndef SECUREC_SNPRINTF_TRUNCATED +#define SECUREC_SNPRINTF_TRUNCATED 1 +#endif + +#if SECUREC_USING_STD_SECURE_LIB +#if defined(_MSC_VER) && _MSC_VER >= 1400 +/* Declare secure functions that are not available in the VS compiler */ +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 1 +#endif +/* VS 2005 have vsnprintf_s function */ +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +/* VS 2005 have vsnprintf_s function Adapt the snprintf_s of the security function */ +#define snprintf_s _snprintf_s +#define SECUREC_ENABLE_SNPRINTF 0 +#endif +/* Before VS 2010 do not have v functions */ +#if _MSC_VER <= 1600 || defined(SECUREC_FOR_V_SCANFS) +#ifndef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSCANF +#define SECUREC_ENABLE_VSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 1 +#endif +#endif + +#else /* MSC VER */ +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 0 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +#define SECUREC_ENABLE_SNPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_MEMMOVE +#define SECUREC_ENABLE_MEMMOVE 0 +#endif +#ifndef SECUREC_ENABLE_MEMCPY +#define SECUREC_ENABLE_MEMCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRCPY +#define SECUREC_ENABLE_STRCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRNCPY +#define SECUREC_ENABLE_STRNCPY 0 +#endif +#ifndef SECUREC_ENABLE_STRCAT +#define SECUREC_ENABLE_STRCAT 0 +#endif +#ifndef SECUREC_ENABLE_STRNCAT +#define SECUREC_ENABLE_STRNCAT 0 +#endif +#ifndef SECUREC_ENABLE_SPRINTF +#define SECUREC_ENABLE_SPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_VSPRINTF +#define SECUREC_ENABLE_VSPRINTF 0 +#endif +#ifndef SECUREC_ENABLE_SSCANF +#define SECUREC_ENABLE_SSCANF 0 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 0 +#endif +#ifndef SECUREC_ENABLE_SCANF +#define SECUREC_ENABLE_SCANF 0 +#endif +#ifndef SECUREC_ENABLE_VSCANF +#define SECUREC_ENABLE_VSCANF 0 +#endif + +#ifndef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif +#ifndef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#ifndef SECUREC_ENABLE_STRTOK +#define SECUREC_ENABLE_STRTOK 0 +#endif +#ifndef SECUREC_ENABLE_GETS +#define SECUREC_ENABLE_GETS 0 +#endif + +#else /* SECUREC USE STD SECURE LIB */ + +#ifndef SECUREC_ENABLE_MEMSET +#define SECUREC_ENABLE_MEMSET 1 +#endif +#ifndef SECUREC_ENABLE_MEMMOVE +#define SECUREC_ENABLE_MEMMOVE 1 +#endif +#ifndef SECUREC_ENABLE_MEMCPY +#define SECUREC_ENABLE_MEMCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRCPY +#define SECUREC_ENABLE_STRCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRNCPY +#define SECUREC_ENABLE_STRNCPY 1 +#endif +#ifndef SECUREC_ENABLE_STRCAT +#define SECUREC_ENABLE_STRCAT 1 +#endif +#ifndef SECUREC_ENABLE_STRNCAT +#define SECUREC_ENABLE_STRNCAT 1 +#endif +#ifndef SECUREC_ENABLE_SPRINTF +#define SECUREC_ENABLE_SPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_VSPRINTF +#define SECUREC_ENABLE_VSPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_SNPRINTF +#define SECUREC_ENABLE_SNPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_VSNPRINTF +#define SECUREC_ENABLE_VSNPRINTF 1 +#endif +#ifndef SECUREC_ENABLE_SSCANF +#define SECUREC_ENABLE_SSCANF 1 +#endif +#ifndef SECUREC_ENABLE_VSSCANF +#define SECUREC_ENABLE_VSSCANF 1 +#endif +#ifndef SECUREC_ENABLE_SCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF 1 +#else +#define SECUREC_ENABLE_SCANF 0 +#endif +#endif +#ifndef SECUREC_ENABLE_VSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_VSCANF 1 +#else +#define SECUREC_ENABLE_VSCANF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_FSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_FSCANF 1 +#else +#define SECUREC_ENABLE_FSCANF 0 +#endif +#endif +#ifndef SECUREC_ENABLE_VFSCANF +#if SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_VFSCANF 1 +#else +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#endif + +#ifndef SECUREC_ENABLE_STRTOK +#define SECUREC_ENABLE_STRTOK 1 +#endif +#ifndef SECUREC_ENABLE_GETS +#define SECUREC_ENABLE_GETS 1 +#endif +#endif /* SECUREC_USE_STD_SECURE_LIB */ + +#if !SECUREC_ENABLE_SCANF_FILE +#if SECUREC_ENABLE_FSCANF +#undef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif +#if SECUREC_ENABLE_VFSCANF +#undef SECUREC_ENABLE_VFSCANF +#define SECUREC_ENABLE_VFSCANF 0 +#endif +#if SECUREC_ENABLE_SCANF +#undef SECUREC_ENABLE_SCANF +#define SECUREC_ENABLE_SCANF 0 +#endif +#if SECUREC_ENABLE_FSCANF +#undef SECUREC_ENABLE_FSCANF +#define SECUREC_ENABLE_FSCANF 0 +#endif + +#endif + +#if SECUREC_IN_KERNEL +#include +#include +#else +#ifndef SECUREC_HAVE_STDIO_H +#define SECUREC_HAVE_STDIO_H 1 +#endif +#ifndef SECUREC_HAVE_STRING_H +#define SECUREC_HAVE_STRING_H 1 +#endif +#ifndef SECUREC_HAVE_STDLIB_H +#define SECUREC_HAVE_STDLIB_H 1 +#endif +#if SECUREC_HAVE_STDIO_H +#include +#endif +#if SECUREC_HAVE_STRING_H +#include +#endif +#if SECUREC_HAVE_STDLIB_H +#include +#endif +#endif + +/* + * If you need high performance, enable the SECUREC_WITH_PERFORMANCE_ADDONS macro, default is enable. + * The macro is automatically closed on the windows platform and linux kernel + */ +#ifndef SECUREC_WITH_PERFORMANCE_ADDONS +#if SECUREC_IN_KERNEL +#define SECUREC_WITH_PERFORMANCE_ADDONS 0 +#else +#define SECUREC_WITH_PERFORMANCE_ADDONS 1 +#endif +#endif + +/* If enable SECUREC_COMPATIBLE_WIN_FORMAT, the output format will be compatible to Windows. */ +#if (defined(_WIN32) || defined(_WIN64) || defined(_MSC_VER)) && !defined(SECUREC_COMPATIBLE_LINUX_FORMAT) +#ifndef SECUREC_COMPATIBLE_WIN_FORMAT +#define SECUREC_COMPATIBLE_WIN_FORMAT +#endif +#endif + +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) +/* On windows platform, can't use optimized function for there is no __builtin_constant_p like function */ +/* If need optimized macro, can define this: define __builtin_constant_p(x) 0 */ +#ifdef SECUREC_WITH_PERFORMANCE_ADDONS +#undef SECUREC_WITH_PERFORMANCE_ADDONS +#define SECUREC_WITH_PERFORMANCE_ADDONS 0 +#endif +#endif + +#if defined(__VXWORKS__) || defined(__vxworks) || defined(__VXWORKS) || defined(_VXWORKS_PLATFORM_) || \ + defined(SECUREC_VXWORKS_VERSION_5_4) +#ifndef SECUREC_VXWORKS_PLATFORM +#define SECUREC_VXWORKS_PLATFORM +#endif +#endif + +/* If enable SECUREC_COMPATIBLE_LINUX_FORMAT, the output format will be compatible to Linux. */ +#if !defined(SECUREC_COMPATIBLE_WIN_FORMAT) && !defined(SECUREC_VXWORKS_PLATFORM) +#ifndef SECUREC_COMPATIBLE_LINUX_FORMAT +#define SECUREC_COMPATIBLE_LINUX_FORMAT +#endif +#endif + +#ifdef SECUREC_COMPATIBLE_LINUX_FORMAT +#ifndef SECUREC_HAVE_STDDEF_H +#define SECUREC_HAVE_STDDEF_H 1 +#endif +/* Some system may no stddef.h */ +#if SECUREC_HAVE_STDDEF_H +#if !SECUREC_IN_KERNEL +#include +#endif +#endif +#endif + +/* + * Add the -DSECUREC_SUPPORT_FORMAT_WARNING=1 compiler option to supoort -Wformat=2. + * Default does not check the format is that the same data type in the actual code. + * In the product is different in the original data type definition of VxWorks and Linux. + */ +#ifndef SECUREC_SUPPORT_FORMAT_WARNING +#define SECUREC_SUPPORT_FORMAT_WARNING 0 +#endif + +#if SECUREC_SUPPORT_FORMAT_WARNING +#define SECUREC_ATTRIBUTE(x, y) __attribute__((format(printf, (x), (y)))) +#else +#define SECUREC_ATTRIBUTE(x, y) +#endif + +/* + * Add the -DSECUREC_SUPPORT_BUILTIN_EXPECT=0 compiler option, if compiler can not support __builtin_expect. + */ +#ifndef SECUREC_SUPPORT_BUILTIN_EXPECT +#define SECUREC_SUPPORT_BUILTIN_EXPECT 1 +#endif + +#if SECUREC_SUPPORT_BUILTIN_EXPECT && defined(__GNUC__) && ((__GNUC__ > 3) || \ + (defined(__GNUC_MINOR__) && (__GNUC__ == 3 && __GNUC_MINOR__ > 3))) +/* + * This is a built-in function that can be used without a declaration, if warning for declaration not found occurred, + * you can add -DSECUREC_NEED_BUILTIN_EXPECT_DECLARE to compiler options + */ +#ifdef SECUREC_NEED_BUILTIN_EXPECT_DECLARE +long __builtin_expect(long exp, long c); +#endif + +#define SECUREC_LIKELY(x) __builtin_expect(!!(x), 1) +#define SECUREC_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else +#define SECUREC_LIKELY(x) (x) +#define SECUREC_UNLIKELY(x) (x) +#endif + +/* Define the max length of the string */ +#ifndef SECUREC_STRING_MAX_LEN +#define SECUREC_STRING_MAX_LEN 0x7fffffffUL +#endif +#define SECUREC_WCHAR_STRING_MAX_LEN (SECUREC_STRING_MAX_LEN / sizeof(wchar_t)) + +/* Add SECUREC_MEM_MAX_LEN for memcpy and memmove */ +#ifndef SECUREC_MEM_MAX_LEN +#define SECUREC_MEM_MAX_LEN 0x7fffffffUL +#endif +#define SECUREC_WCHAR_MEM_MAX_LEN (SECUREC_MEM_MAX_LEN / sizeof(wchar_t)) + +#if SECUREC_STRING_MAX_LEN > 0x7fffffffUL +#error "max string is 2G" +#endif + +#if (defined(__GNUC__) && defined(__SIZEOF_POINTER__)) +#if (__SIZEOF_POINTER__ != 4) && (__SIZEOF_POINTER__ != 8) +#error "unsupported system" +#endif +#endif + +#if defined(_WIN64) || defined(WIN64) || defined(__LP64__) || defined(_LP64) +#define SECUREC_ON_64BITS +#endif + +#if (!defined(SECUREC_ON_64BITS) && defined(__GNUC__) && defined(__SIZEOF_POINTER__)) +#if __SIZEOF_POINTER__ == 8 +#define SECUREC_ON_64BITS +#endif +#endif + +#if defined(__SVR4) || defined(__svr4__) +#define SECUREC_ON_SOLARIS +#endif + +#if (defined(__hpux) || defined(_AIX) || defined(SECUREC_ON_SOLARIS)) +#define SECUREC_ON_UNIX +#endif + +/* + * Codes should run under the macro SECUREC_COMPATIBLE_LINUX_FORMAT in unknown system on default, + * and strtold. + * The function strtold is referenced first at ISO9899:1999(C99), and some old compilers can + * not support these functions. Here provides a macro to open these functions: + * SECUREC_SUPPORT_STRTOLD -- If defined, strtold will be used + */ +#ifndef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 0 +#if (defined(SECUREC_COMPATIBLE_LINUX_FORMAT)) +#if defined(__USE_ISOC99) || \ + (defined(_AIX) && defined(_ISOC99_SOURCE)) || \ + (defined(__hpux) && defined(__ia64)) || \ + (defined(SECUREC_ON_SOLARIS) && (!defined(_STRICT_STDC) && !defined(__XOPEN_OR_POSIX)) || \ + defined(_STDC_C99) || defined(__EXTENSIONS__)) +#undef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 1 +#endif +#endif +#if ((defined(SECUREC_WRLINUX_BELOW4) || defined(_WRLINUX_BELOW4_))) +#undef SECUREC_SUPPORT_STRTOLD +#define SECUREC_SUPPORT_STRTOLD 0 +#endif +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS + +#ifndef SECUREC_TWO_MIN +#define SECUREC_TWO_MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif + +/* This macro do not check buffer overlap by default */ +#define SECUREC_MEMCPY_SM(dest, destMax, src, count) \ + (!(((size_t)(destMax) == 0) || \ + (((unsigned long long)(destMax) & (unsigned long long)(-2)) > SECUREC_MEM_MAX_LEN) || \ + ((size_t)(count) > (size_t)(destMax)) || ((void *)(dest)) == NULL || ((const void *)(src) == NULL)) ? \ + (memcpy((dest), (src), (count)), EOK) : \ + (memcpy_s((dest), (destMax), (src), (count)))) + +#define SECUREC_MEMSET_SM(dest, destMax, c, count) \ + (!((((unsigned long long)(destMax) & (unsigned long long)(-2)) > SECUREC_MEM_MAX_LEN) || \ + ((void *)(dest) == NULL) || ((size_t)(count) > (size_t)(destMax))) ? \ + (memset((dest), (c), (count)), EOK) : \ + (memset_s((dest), (destMax), (c), (count)))) + +#endif +#endif diff --git a/msmonitor/plugin/third_party/securec/src/memcpy_s.c b/msmonitor/plugin/third_party/securec/src/memcpy_s.c new file mode 100644 index 0000000000000000000000000000000000000000..a7fd48748e50a7180c2afd8a1def9b05180eb8bc --- /dev/null +++ b/msmonitor/plugin/third_party/securec/src/memcpy_s.c @@ -0,0 +1,555 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2014-2021. All rights reserved. + * Licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * Description: memcpy_s function + * Create: 2014-02-25 + */ +/* + * [Standardize-exceptions] Use unsafe function: Portability + * [reason] Use unsafe function to implement security function to maintain platform compatibility. + * And sufficient input validation is performed before calling + */ + +#include "securecutil.h" + +#if SECUREC_WITH_PERFORMANCE_ADDONS +#ifndef SECUREC_MEMCOPY_THRESHOLD_SIZE +#define SECUREC_MEMCOPY_THRESHOLD_SIZE 64UL +#endif + +#define SECUREC_SMALL_MEM_COPY(dest, src, count) do { \ + if (SECUREC_ADDR_ALIGNED_8(dest) && SECUREC_ADDR_ALIGNED_8(src)) { \ + /* Use struct assignment */ \ + switch (count) { \ + case 1: \ + *(unsigned char *)(dest) = *(const unsigned char *)(src); \ + break; \ + case 2: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 2); \ + break; \ + case 3: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 3); \ + break; \ + case 4: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 4); \ + break; \ + case 5: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 5); \ + break; \ + case 6: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 6); \ + break; \ + case 7: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 7); \ + break; \ + case 8: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 8); \ + break; \ + case 9: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 9); \ + break; \ + case 10: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 10); \ + break; \ + case 11: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 11); \ + break; \ + case 12: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 12); \ + break; \ + case 13: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 13); \ + break; \ + case 14: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 14); \ + break; \ + case 15: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 15); \ + break; \ + case 16: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 16); \ + break; \ + case 17: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 17); \ + break; \ + case 18: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 18); \ + break; \ + case 19: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 19); \ + break; \ + case 20: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 20); \ + break; \ + case 21: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 21); \ + break; \ + case 22: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 22); \ + break; \ + case 23: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 23); \ + break; \ + case 24: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 24); \ + break; \ + case 25: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 25); \ + break; \ + case 26: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 26); \ + break; \ + case 27: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 27); \ + break; \ + case 28: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 28); \ + break; \ + case 29: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 29); \ + break; \ + case 30: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 30); \ + break; \ + case 31: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 31); \ + break; \ + case 32: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 32); \ + break; \ + case 33: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 33); \ + break; \ + case 34: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 34); \ + break; \ + case 35: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 35); \ + break; \ + case 36: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 36); \ + break; \ + case 37: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 37); \ + break; \ + case 38: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 38); \ + break; \ + case 39: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 39); \ + break; \ + case 40: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 40); \ + break; \ + case 41: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 41); \ + break; \ + case 42: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 42); \ + break; \ + case 43: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 43); \ + break; \ + case 44: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 44); \ + break; \ + case 45: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 45); \ + break; \ + case 46: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 46); \ + break; \ + case 47: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 47); \ + break; \ + case 48: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 48); \ + break; \ + case 49: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 49); \ + break; \ + case 50: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 50); \ + break; \ + case 51: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 51); \ + break; \ + case 52: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 52); \ + break; \ + case 53: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 53); \ + break; \ + case 54: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 54); \ + break; \ + case 55: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 55); \ + break; \ + case 56: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 56); \ + break; \ + case 57: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 57); \ + break; \ + case 58: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 58); \ + break; \ + case 59: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 59); \ + break; \ + case 60: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 60); \ + break; \ + case 61: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 61); \ + break; \ + case 62: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 62); \ + break; \ + case 63: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 63); \ + break; \ + case 64: \ + SECUREC_COPY_VALUE_BY_STRUCT((dest), (src), 64); \ + break; \ + default: \ + /* Do nothing */ \ + break; \ + } /* END switch */ \ + } else { \ + unsigned char *tmpDest_ = (unsigned char *)(dest); \ + const unsigned char *tmpSrc_ = (const unsigned char *)(src); \ + switch (count) { \ + case 64: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 63: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 62: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 61: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 60: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 59: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 58: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 57: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 56: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 55: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 54: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 53: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 52: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 51: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 50: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 49: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 48: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 47: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 46: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 45: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 44: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 43: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 42: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 41: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 40: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 39: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 38: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 37: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 36: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 35: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 34: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 33: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 32: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 31: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 30: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 29: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 28: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 27: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 26: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 25: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 24: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 23: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 22: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 21: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 20: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 19: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 18: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 17: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 16: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 15: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 14: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 13: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 12: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 11: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 10: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 9: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 8: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 7: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 6: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 5: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 4: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 3: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 2: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + case 1: \ + *(tmpDest_++) = *(tmpSrc_++); \ + /* fall-through */ /* FALLTHRU */ \ + default: \ + /* Do nothing */ \ + break; \ + } \ + } \ +} SECUREC_WHILE_ZERO + +/* + * Performance optimization + */ +#define SECUREC_MEMCPY_OPT(dest, src, count) do { \ + if ((count) > SECUREC_MEMCOPY_THRESHOLD_SIZE) { \ + SECUREC_MEMCPY_WARP_OPT((dest), (src), (count)); \ + } else { \ + SECUREC_SMALL_MEM_COPY((dest), (src), (count)); \ + } \ +} SECUREC_WHILE_ZERO +#endif + +/* + * Handling errors + */ +SECUREC_INLINE errno_t SecMemcpyError(void *dest, size_t destMax, const void *src, size_t count) +{ + if (destMax == 0 || destMax > SECUREC_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("memcpy_s"); + return ERANGE; + } + if (dest == NULL || src == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("memcpy_s"); + if (dest != NULL) { + (void)SECUREC_MEMSET_FUNC_OPT(dest, 0, destMax); + return EINVAL_AND_RESET; + } + return EINVAL; + } + if (count > destMax) { + (void)SECUREC_MEMSET_FUNC_OPT(dest, 0, destMax); + SECUREC_ERROR_INVALID_RANGE("memcpy_s"); + return ERANGE_AND_RESET; + } + if (SECUREC_MEMORY_IS_OVERLAP(dest, src, count)) { + (void)SECUREC_MEMSET_FUNC_OPT(dest, 0, destMax); + SECUREC_ERROR_BUFFER_OVERLAP("memcpy_s"); + return EOVERLAP_AND_RESET; + } + /* Count is 0 or dest equal src also ret EOK */ + return EOK; +} + +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) + /* + * The fread API in windows will call memcpy_s and pass 0xffffffff to destMax. + * To avoid the failure of fread, we don't check desMax limit. + */ +#define SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count) (SECUREC_LIKELY((count) <= (destMax) && \ + (dest) != NULL && (src) != NULL && \ + (count) > 0 && SECUREC_MEMORY_NO_OVERLAP((dest), (src), (count)))) +#else +#define SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count) (SECUREC_LIKELY((count) <= (destMax) && \ + (dest) != NULL && (src) != NULL && (destMax) <= SECUREC_MEM_MAX_LEN && \ + (count) > 0 && SECUREC_MEMORY_NO_OVERLAP((dest), (src), (count)))) +#endif + +/* + * + * The memcpy_s function copies n characters from the object pointed to by src into the object pointed to by dest + * + * + * dest Destination buffer. + * destMax Size of the destination buffer. + * src Buffer to copy from. + * count Number of characters to copy + * + * + * dest buffer is updated. + * + * + * EOK Success + * EINVAL dest is NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * EINVAL_AND_RESET dest != NULL and src is NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * ERANGE destMax > SECUREC_MEM_MAX_LEN or destMax is 0 + * ERANGE_AND_RESET count > destMax and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * and dest != NULL and src != NULL + * EOVERLAP_AND_RESET dest buffer and source buffer are overlapped and + * count <= destMax destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN and dest != NULL + * and src != NULL and dest != src + * + * if an error occurred, dest will be filled with 0. + * If the source and destination overlap, the behavior of memcpy_s is undefined. + * Use memmove_s to handle overlapping regions. + */ +errno_t memcpy_s(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count)) { + SECUREC_MEMCPY_WARP_OPT(dest, src, count); + return EOK; + } + /* Meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} + +#if SECUREC_EXPORT_KERNEL_SYMBOL +EXPORT_SYMBOL(memcpy_s); +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS +/* + * Performance optimization + */ +errno_t memcpy_sOptAsm(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_MEMCPY_PARAM_OK(dest, destMax, src, count)) { + SECUREC_MEMCPY_OPT(dest, src, count); + return EOK; + } + /* Meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} + +/* Trim judgement on "destMax <= SECUREC_MEM_MAX_LEN" */ +errno_t memcpy_sOptTc(void *dest, size_t destMax, const void *src, size_t count) +{ + if (SECUREC_LIKELY(count <= destMax && dest != NULL && src != NULL && \ + count > 0 && SECUREC_MEMORY_NO_OVERLAP((dest), (src), (count)))) { + SECUREC_MEMCPY_OPT(dest, src, count); + return EOK; + } + /* Meet some runtime violation, return error code */ + return SecMemcpyError(dest, destMax, src, count); +} +#endif + diff --git a/msmonitor/plugin/third_party/securec/src/memset_s.c b/msmonitor/plugin/third_party/securec/src/memset_s.c new file mode 100644 index 0000000000000000000000000000000000000000..d9a657fd326af60ec1195b226aa762855042299b --- /dev/null +++ b/msmonitor/plugin/third_party/securec/src/memset_s.c @@ -0,0 +1,510 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2014-2021. All rights reserved. + * Licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * Description: memset_s function + * Create: 2014-02-25 + */ +/* + * [Standardize-exceptions] Use unsafe function: Portability + * [reason] Use unsafe function to implement security function to maintain platform compatibility. + * And sufficient input validation is performed before calling + */ + +#include "securecutil.h" + +#define SECUREC_MEMSET_PARAM_OK(dest, destMax, count) (SECUREC_LIKELY((destMax) <= SECUREC_MEM_MAX_LEN && \ + (dest) != NULL && (count) <= (destMax))) + +#if SECUREC_WITH_PERFORMANCE_ADDONS + +/* Use union to clear strict-aliasing warning */ +typedef union { + SecStrBuf32 buf32; + SecStrBuf31 buf31; + SecStrBuf30 buf30; + SecStrBuf29 buf29; + SecStrBuf28 buf28; + SecStrBuf27 buf27; + SecStrBuf26 buf26; + SecStrBuf25 buf25; + SecStrBuf24 buf24; + SecStrBuf23 buf23; + SecStrBuf22 buf22; + SecStrBuf21 buf21; + SecStrBuf20 buf20; + SecStrBuf19 buf19; + SecStrBuf18 buf18; + SecStrBuf17 buf17; + SecStrBuf16 buf16; + SecStrBuf15 buf15; + SecStrBuf14 buf14; + SecStrBuf13 buf13; + SecStrBuf12 buf12; + SecStrBuf11 buf11; + SecStrBuf10 buf10; + SecStrBuf9 buf9; + SecStrBuf8 buf8; + SecStrBuf7 buf7; + SecStrBuf6 buf6; + SecStrBuf5 buf5; + SecStrBuf4 buf4; + SecStrBuf3 buf3; + SecStrBuf2 buf2; +} SecStrBuf32Union; +/* C standard initializes the first member of the consortium. */ +static const SecStrBuf32 g_allZero = {{ + 0U, 0U, 0U, 0U, 0U, 0U, 0U, 0U, + 0U, 0U, 0U, 0U, 0U, 0U, 0U, 0U, + 0U, 0U, 0U, 0U, 0U, 0U, 0U, 0U, + 0U, 0U, 0U, 0U, 0U, 0U, 0U, 0U +}}; +static const SecStrBuf32 g_allFF = {{ + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF +}}; + +/* Clear conversion warning strict aliasing" */ +SECUREC_INLINE const SecStrBuf32Union *SecStrictAliasingCast(const SecStrBuf32 *buf) +{ + return (const SecStrBuf32Union *)buf; +} + +#ifndef SECUREC_MEMSET_THRESHOLD_SIZE +#define SECUREC_MEMSET_THRESHOLD_SIZE 32UL +#endif + +#define SECUREC_UNALIGNED_SET(dest, c, count) do { \ + unsigned char *pDest_ = (unsigned char *)(dest); \ + switch (count) { \ + case 32: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 31: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 30: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 29: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 28: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 27: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 26: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 25: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 24: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 23: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 22: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 21: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 20: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 19: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 18: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 17: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 16: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 15: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 14: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 13: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 12: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 11: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 10: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 9: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 8: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 7: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 6: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 5: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 4: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 3: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 2: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + case 1: \ + *(pDest_++) = (unsigned char)(c); \ + /* fall-through */ /* FALLTHRU */ \ + default: \ + /* Do nothing */ \ + break; \ + } \ +} SECUREC_WHILE_ZERO + +#define SECUREC_SET_VALUE_BY_STRUCT(dest, dataName, n) do { \ + *(SecStrBuf##n *)(dest) = *(const SecStrBuf##n *)(&((SecStrictAliasingCast(&(dataName)))->buf##n)); \ +} SECUREC_WHILE_ZERO + +#define SECUREC_ALIGNED_SET_OPT_ZERO_FF(dest, c, count) do { \ + switch (c) { \ + case 0: \ + switch (count) { \ + case 1: \ + *(unsigned char *)(dest) = (unsigned char)0; \ + break; \ + case 2: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 2); \ + break; \ + case 3: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 3); \ + break; \ + case 4: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 4); \ + break; \ + case 5: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 5); \ + break; \ + case 6: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 6); \ + break; \ + case 7: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 7); \ + break; \ + case 8: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 8); \ + break; \ + case 9: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 9); \ + break; \ + case 10: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 10); \ + break; \ + case 11: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 11); \ + break; \ + case 12: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 12); \ + break; \ + case 13: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 13); \ + break; \ + case 14: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 14); \ + break; \ + case 15: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 15); \ + break; \ + case 16: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 16); \ + break; \ + case 17: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 17); \ + break; \ + case 18: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 18); \ + break; \ + case 19: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 19); \ + break; \ + case 20: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 20); \ + break; \ + case 21: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 21); \ + break; \ + case 22: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 22); \ + break; \ + case 23: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 23); \ + break; \ + case 24: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 24); \ + break; \ + case 25: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 25); \ + break; \ + case 26: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 26); \ + break; \ + case 27: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 27); \ + break; \ + case 28: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 28); \ + break; \ + case 29: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 29); \ + break; \ + case 30: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 30); \ + break; \ + case 31: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 31); \ + break; \ + case 32: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allZero, 32); \ + break; \ + default: \ + /* Do nothing */ \ + break; \ + } \ + break; \ + case 0xFF: \ + switch (count) { \ + case 1: \ + *(unsigned char *)(dest) = (unsigned char)0xffU; \ + break; \ + case 2: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 2); \ + break; \ + case 3: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 3); \ + break; \ + case 4: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 4); \ + break; \ + case 5: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 5); \ + break; \ + case 6: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 6); \ + break; \ + case 7: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 7); \ + break; \ + case 8: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 8); \ + break; \ + case 9: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 9); \ + break; \ + case 10: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 10); \ + break; \ + case 11: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 11); \ + break; \ + case 12: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 12); \ + break; \ + case 13: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 13); \ + break; \ + case 14: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 14); \ + break; \ + case 15: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 15); \ + break; \ + case 16: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 16); \ + break; \ + case 17: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 17); \ + break; \ + case 18: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 18); \ + break; \ + case 19: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 19); \ + break; \ + case 20: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 20); \ + break; \ + case 21: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 21); \ + break; \ + case 22: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 22); \ + break; \ + case 23: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 23); \ + break; \ + case 24: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 24); \ + break; \ + case 25: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 25); \ + break; \ + case 26: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 26); \ + break; \ + case 27: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 27); \ + break; \ + case 28: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 28); \ + break; \ + case 29: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 29); \ + break; \ + case 30: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 30); \ + break; \ + case 31: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 31); \ + break; \ + case 32: \ + SECUREC_SET_VALUE_BY_STRUCT((dest), g_allFF, 32); \ + break; \ + default: \ + /* Do nothing */ \ + break; \ + } \ + break; \ + default: \ + SECUREC_UNALIGNED_SET((dest), (c), (count)); \ + break; \ + } /* END switch */ \ +} SECUREC_WHILE_ZERO + +#define SECUREC_SMALL_MEM_SET(dest, c, count) do { \ + if (SECUREC_ADDR_ALIGNED_8((dest))) { \ + SECUREC_ALIGNED_SET_OPT_ZERO_FF((dest), (c), (count)); \ + } else { \ + SECUREC_UNALIGNED_SET((dest), (c), (count)); \ + } \ +} SECUREC_WHILE_ZERO + +/* + * Performance optimization + */ +#define SECUREC_MEMSET_OPT(dest, c, count) do { \ + if ((count) > SECUREC_MEMSET_THRESHOLD_SIZE) { \ + SECUREC_MEMSET_PREVENT_DSE((dest), (c), (count)); \ + } else { \ + SECUREC_SMALL_MEM_SET((dest), (c), (count)); \ + } \ +} SECUREC_WHILE_ZERO +#endif + +/* + * Handling errors + */ +SECUREC_INLINE errno_t SecMemsetError(void *dest, size_t destMax, int c) +{ + /* Check destMax is 0 compatible with _sp macro */ + if (destMax == 0 || destMax > SECUREC_MEM_MAX_LEN) { + SECUREC_ERROR_INVALID_RANGE("memset_s"); + return ERANGE; + } + if (dest == NULL) { + SECUREC_ERROR_INVALID_PARAMTER("memset_s"); + return EINVAL; + } + SECUREC_MEMSET_PREVENT_DSE(dest, c, destMax); /* Set entire buffer to value c */ + SECUREC_ERROR_INVALID_RANGE("memset_s"); + return ERANGE_AND_RESET; +} + +/* + * + * The memset_s function copies the value of c (converted to an unsigned char) + * into each of the first count characters of the object pointed to by dest. + * + * + * dest Pointer to destination. + * destMax The size of the buffer. + * c Character to set. + * count Number of characters. + * + * + * dest buffer is updated. + * + * + * EOK Success + * EINVAL dest == NULL and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN + * ERANGE destMax > SECUREC_MEM_MAX_LEN or (destMax is 0 and count > destMax) + * ERANGE_AND_RESET count > destMax and destMax != 0 and destMax <= SECUREC_MEM_MAX_LEN and dest != NULL + * + * if return ERANGE_AND_RESET then fill dest to c ,fill length is destMax + */ +errno_t memset_s(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_MEMSET_PARAM_OK(dest, destMax, count)) { + SECUREC_MEMSET_PREVENT_DSE(dest, c, count); + return EOK; + } + /* Meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c); +} + +#if SECUREC_EXPORT_KERNEL_SYMBOL +EXPORT_SYMBOL(memset_s); +#endif + +#if SECUREC_WITH_PERFORMANCE_ADDONS +/* + * Performance optimization + */ +errno_t memset_sOptAsm(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_MEMSET_PARAM_OK(dest, destMax, count)) { + SECUREC_MEMSET_OPT(dest, c, count); + return EOK; + } + /* Meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c); +} + +/* + * Performance optimization, trim judgement on "destMax <= SECUREC_MEM_MAX_LEN" + */ +errno_t memset_sOptTc(void *dest, size_t destMax, int c, size_t count) +{ + if (SECUREC_LIKELY(count <= destMax && dest != NULL)) { + SECUREC_MEMSET_OPT(dest, c, count); + return EOK; + } + /* Meet some runtime violation, return error code */ + return SecMemsetError(dest, destMax, c); +} +#endif + diff --git a/msmonitor/plugin/third_party/securec/src/securecutil.c b/msmonitor/plugin/third_party/securec/src/securecutil.c new file mode 100644 index 0000000000000000000000000000000000000000..0053a72cfab51526702fecc78d1cbe4616e68abb --- /dev/null +++ b/msmonitor/plugin/third_party/securec/src/securecutil.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2014-2021. All rights reserved. + * Licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * Description: Provides internal functions used by this library, such as memory + * copy and memory move. Besides, include some helper function for + * printf family API, such as SecVsnprintfImpl + * Create: 2014-02-25 + */ + +/* Avoid duplicate header files,not include securecutil.h */ +#include "securecutil.h" + +#if defined(ANDROID) && !defined(SECUREC_CLOSE_ANDROID_HANDLE) && (SECUREC_HAVE_WCTOMB || SECUREC_HAVE_MBTOWC) +#include +#if SECUREC_HAVE_WCTOMB +/* + * Convert wide characters to narrow multi-bytes + */ +int wctomb(char *s, wchar_t wc) +{ + return (int)wcrtomb(s, wc, NULL); +} +#endif + +#if SECUREC_HAVE_MBTOWC +/* + * Converting narrow multi-byte characters to wide characters + * mbrtowc returns -1 or -2 upon failure, unlike mbtowc, which only returns -1 + * When the return value is less than zero, we treat it as a failure + */ +int mbtowc(wchar_t *pwc, const char *s, size_t n) +{ + return (int)mbrtowc(pwc, s, n, NULL); +} +#endif +#endif + +/* The V100R001C01 version num is 0x5 (High 8 bits) */ +#define SECUREC_C_VERSION 0x500U +#define SECUREC_SPC_VERSION 0x10U +#define SECUREC_VERSION_STR "1.1.16" + +/* + * Get version string and version number. + * The rules for version number are as follows: + * 1) SPC verNumber<->verStr like: + * 0x201<->C01 + * 0x202<->C01SPC001 Redefine numbers after this version + * 0x502<->C01SPC002 + * 0x503<->C01SPC003 + * ... + * 0X50a<->SPC010 + * 0X50b<->SPC011 + * ... + * 0x700<->C02 + * 0x701<->C01SPC001 + * 0x702<->C02SPC002 + * ... + * 2) CP verNumber<->verStr like: + * 0X601<->CP0001 + * 0X602<->CP0002 + * ... + */ +const char *GetHwSecureCVersion(unsigned short *verNumber) +{ + if (verNumber != NULL) { + *verNumber = (unsigned short)(SECUREC_C_VERSION | SECUREC_SPC_VERSION); + } + return SECUREC_VERSION_STR; +} +#if SECUREC_EXPORT_KERNEL_SYMBOL +EXPORT_SYMBOL(GetHwSecureCVersion); +#endif + diff --git a/msmonitor/plugin/third_party/securec/src/securecutil.h b/msmonitor/plugin/third_party/securec/src/securecutil.h new file mode 100644 index 0000000000000000000000000000000000000000..7e3bd691f9ece9decd2fcb3c239697c806597246 --- /dev/null +++ b/msmonitor/plugin/third_party/securec/src/securecutil.h @@ -0,0 +1,574 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2014-2021. All rights reserved. + * Licensed under Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, + * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, + * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. + * See the Mulan PSL v2 for more details. + * Description: Define macro, data struct, and declare internal used function prototype, + * which is used by secure functions. + * Create: 2014-02-25 + */ + +#ifndef SECURECUTIL_H_46C86578_F8FF_4E49_8E64_9B175241761F +#define SECURECUTIL_H_46C86578_F8FF_4E49_8E64_9B175241761F +#include "securec.h" + +#if (defined(_MSC_VER)) && (_MSC_VER >= 1400) +/* Shield compilation alerts using discarded functions and Constant expression to maximize code compatibility */ +#define SECUREC_MASK_MSVC_CRT_WARNING __pragma(warning(push)) \ + __pragma(warning(disable : 4996 4127)) +#define SECUREC_END_MASK_MSVC_CRT_WARNING __pragma(warning(pop)) +#else +#define SECUREC_MASK_MSVC_CRT_WARNING +#define SECUREC_END_MASK_MSVC_CRT_WARNING +#endif +#define SECUREC_WHILE_ZERO SECUREC_MASK_MSVC_CRT_WARNING while (0) SECUREC_END_MASK_MSVC_CRT_WARNING + +/* Automatically identify the platform that supports strnlen function, and use this function to improve performance */ +#ifndef SECUREC_HAVE_STRNLEN +#if (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE >= 700) || (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 200809L) +#if SECUREC_IN_KERNEL +#define SECUREC_HAVE_STRNLEN 0 +#else +#if defined(__GLIBC__) && __GLIBC__ >= 2 && defined(__GLIBC_MINOR__) && __GLIBC_MINOR__ >= 10 +#define SECUREC_HAVE_STRNLEN 1 +#else +#define SECUREC_HAVE_STRNLEN 0 +#endif +#endif +#else +#define SECUREC_HAVE_STRNLEN 0 +#endif +#endif + +#if SECUREC_IN_KERNEL +/* In kernel disable functions */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 0 +#endif +#ifndef SECUREC_ENABLE_SCANF_FLOAT +#define SECUREC_ENABLE_SCANF_FLOAT 0 +#endif +#ifndef SECUREC_ENABLE_SPRINTF_FLOAT +#define SECUREC_ENABLE_SPRINTF_FLOAT 0 +#endif +#ifndef SECUREC_HAVE_MBTOWC +#define SECUREC_HAVE_MBTOWC 0 +#endif +#ifndef SECUREC_HAVE_WCTOMB +#define SECUREC_HAVE_WCTOMB 0 +#endif +#ifndef SECUREC_HAVE_WCHART +#define SECUREC_HAVE_WCHART 0 +#endif +#else /* Not in kernel */ +/* Systems that do not support file, can define this macro to 0. */ +#ifndef SECUREC_ENABLE_SCANF_FILE +#define SECUREC_ENABLE_SCANF_FILE 1 +#endif +#ifndef SECUREC_ENABLE_SCANF_FLOAT +#define SECUREC_ENABLE_SCANF_FLOAT 1 +#endif +/* Systems that do not support float, can define this macro to 0. */ +#ifndef SECUREC_ENABLE_SPRINTF_FLOAT +#define SECUREC_ENABLE_SPRINTF_FLOAT 1 +#endif +#ifndef SECUREC_HAVE_MBTOWC +#define SECUREC_HAVE_MBTOWC 1 +#endif +#ifndef SECUREC_HAVE_WCTOMB +#define SECUREC_HAVE_WCTOMB 1 +#endif +#ifndef SECUREC_HAVE_WCHART +#define SECUREC_HAVE_WCHART 1 +#endif +#endif + +#ifndef SECUREC_ENABLE_INLINE +#define SECUREC_ENABLE_INLINE 0 +#endif + +#ifndef SECUREC_INLINE +#if SECUREC_ENABLE_INLINE +#define SECUREC_INLINE static inline +#else +#define SECUREC_INLINE static +#endif +#endif + +#ifndef SECUREC_WARP_OUTPUT +#if SECUREC_IN_KERNEL +#define SECUREC_WARP_OUTPUT 1 +#else +#define SECUREC_WARP_OUTPUT 0 +#endif +#endif + +#ifndef SECUREC_STREAM_STDIN +#define SECUREC_STREAM_STDIN stdin +#endif + +#define SECUREC_MUL_SIXTEEN(x) ((x) << 4U) +#define SECUREC_MUL_EIGHT(x) ((x) << 3U) +#define SECUREC_MUL_TEN(x) ((((x) << 2U) + (x)) << 1U) +/* Limited format input and output width, use signed integer */ +#define SECUREC_MAX_WIDTH_LEN_DIV_TEN 21474836 +#define SECUREC_MAX_WIDTH_LEN (SECUREC_MAX_WIDTH_LEN_DIV_TEN * 10) +/* Is the x multiplied by 10 greater than */ +#define SECUREC_MUL_TEN_ADD_BEYOND_MAX(x) (((x) > SECUREC_MAX_WIDTH_LEN_DIV_TEN)) + +#define SECUREC_FLOAT_BUFSIZE (309 + 40) /* Max length of double value */ +#define SECUREC_FLOAT_BUFSIZE_LB (4932 + 40) /* Max length of long double value */ +#define SECUREC_FLOAT_DEFAULT_PRECISION 6 + +/* This macro does not handle pointer equality or integer overflow */ +#define SECUREC_MEMORY_NO_OVERLAP(dest, src, count) \ + (((src) < (dest) && ((const char *)(src) + (count)) <= (char *)(dest)) || \ + ((dest) < (src) && ((char *)(dest) + (count)) <= (const char *)(src))) + +#define SECUREC_MEMORY_IS_OVERLAP(dest, src, count) \ + (((src) < (dest) && ((const char *)(src) + (count)) > (char *)(dest)) || \ + ((dest) < (src) && ((char *)(dest) + (count)) > (const char *)(src))) + +/* + * Check whether the strings overlap, len is the length of the string not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_STRING_NO_OVERLAP(dest, src, len) \ + (((src) < (dest) && ((src) + (len)) < (dest)) || \ + ((dest) < (src) && ((dest) + (len)) < (src))) + +/* + * Check whether the strings overlap for strcpy wcscpy function, dest len and src Len are not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_STRING_IS_OVERLAP(dest, src, len) \ + (((src) < (dest) && ((src) + (len)) >= (dest)) || \ + ((dest) < (src) && ((dest) + (len)) >= (src))) + +/* + * Check whether the strings overlap for strcat wcscat function, dest len and src Len are not include terminator + * Length is related to data type char or wchar , do not force conversion of types + */ +#define SECUREC_CAT_STRING_IS_OVERLAP(dest, destLen, src, srcLen) \ + (((dest) < (src) && ((dest) + (destLen) + (srcLen)) >= (src)) || \ + ((src) < (dest) && ((src) + (srcLen)) >= (dest))) + +#if SECUREC_HAVE_STRNLEN +#define SECUREC_CALC_STR_LEN(str, maxLen, outLen) do { \ + *(outLen) = strnlen((str), (maxLen)); \ +} SECUREC_WHILE_ZERO +#define SECUREC_CALC_STR_LEN_OPT(str, maxLen, outLen) do { \ + if ((maxLen) > 8) { \ + /* Optimization or len less then 8 */ \ + if (*((str) + 0) == '\0') { \ + *(outLen) = 0; \ + } else if (*((str) + 1) == '\0') { \ + *(outLen) = 1; \ + } else if (*((str) + 2) == '\0') { \ + *(outLen) = 2; \ + } else if (*((str) + 3) == '\0') { \ + *(outLen) = 3; \ + } else if (*((str) + 4) == '\0') { \ + *(outLen) = 4; \ + } else if (*((str) + 5) == '\0') { \ + *(outLen) = 5; \ + } else if (*((str) + 6) == '\0') { \ + *(outLen) = 6; \ + } else if (*((str) + 7) == '\0') { \ + *(outLen) = 7; \ + } else if (*((str) + 8) == '\0') { \ + /* Optimization with a length of 8 */ \ + *(outLen) = 8; \ + } else { \ + /* The offset is 8 because the performance of 8 byte alignment is high */ \ + *(outLen) = 8 + strnlen((str) + 8, (maxLen) - 8); \ + } \ + } else { \ + SECUREC_CALC_STR_LEN((str), (maxLen), (outLen)); \ + } \ +} SECUREC_WHILE_ZERO +#else +#define SECUREC_CALC_STR_LEN(str, maxLen, outLen) do { \ + const char *strEnd_ = (const char *)(str); \ + size_t availableSize_ = (size_t)(maxLen); \ + while (availableSize_ > 0 && *strEnd_ != '\0') { \ + --availableSize_; \ + ++strEnd_; \ + } \ + *(outLen) = (size_t)(strEnd_ - (str)); \ +} SECUREC_WHILE_ZERO +#define SECUREC_CALC_STR_LEN_OPT SECUREC_CALC_STR_LEN +#endif + +#define SECUREC_CALC_WSTR_LEN(str, maxLen, outLen) do { \ + const wchar_t *strEnd_ = (const wchar_t *)(str); \ + size_t len_ = 0; \ + while (len_ < (maxLen) && *strEnd_ != L'\0') { \ + ++len_; \ + ++strEnd_; \ + } \ + *(outLen) = len_; \ +} SECUREC_WHILE_ZERO + +/* + * Performance optimization, product may disable inline function. + * Using function pointer for MEMSET to prevent compiler optimization when cleaning up memory. + */ +#ifdef SECUREC_USE_ASM +#define SECUREC_MEMSET_FUNC_OPT memset_opt +#define SECUREC_MEMCPY_FUNC_OPT memcpy_opt +#else +#define SECUREC_MEMSET_FUNC_OPT memset +#define SECUREC_MEMCPY_FUNC_OPT memcpy +#endif + +#define SECUREC_MEMCPY_WARP_OPT(dest, src, count) (void)SECUREC_MEMCPY_FUNC_OPT((dest), (src), (count)) + +#ifndef SECUREC_MEMSET_BARRIER +#if defined(__GNUC__) +/* Can be turned off for scenarios that do not use memory barrier */ +#define SECUREC_MEMSET_BARRIER 1 +#else +#define SECUREC_MEMSET_BARRIER 0 +#endif +#endif + +#ifndef SECUREC_MEMSET_INDIRECT_USE +/* Can be turned off for scenarios that do not allow pointer calls */ +#define SECUREC_MEMSET_INDIRECT_USE 1 +#endif + +#if SECUREC_MEMSET_BARRIER +#define SECUREC_MEMORY_BARRIER(dest) __asm__ __volatile__("": : "r"(dest) : "memory") +#else +#define SECUREC_MEMORY_BARRIER(dest) +#endif + +#if SECUREC_MEMSET_BARRIER +#define SECUREC_MEMSET_PREVENT_DSE(dest, value, count) do { \ + (void)SECUREC_MEMSET_FUNC_OPT(dest, value, count); \ + SECUREC_MEMORY_BARRIER(dest); \ +} SECUREC_WHILE_ZERO +#elif SECUREC_MEMSET_INDIRECT_USE +#define SECUREC_MEMSET_PREVENT_DSE(dest, value, count) do { \ + void *(* const volatile fn_)(void *s_, int c_, size_t n_) = SECUREC_MEMSET_FUNC_OPT; \ + (void)(*fn_)((dest), (value), (count)); \ +} SECUREC_WHILE_ZERO +#else +#define SECUREC_MEMSET_PREVENT_DSE(dest, value, count) (void)SECUREC_MEMSET_FUNC_OPT((dest), (value), (count)) +#endif + +#ifdef SECUREC_FORMAT_OUTPUT_INPUT +#if defined(SECUREC_COMPATIBLE_WIN_FORMAT) || defined(__ARMCC_VERSION) +typedef __int64 SecInt64; +typedef unsigned __int64 SecUnsignedInt64; +#if defined(__ARMCC_VERSION) +typedef unsigned int SecUnsignedInt32; +#else +typedef unsigned __int32 SecUnsignedInt32; +#endif +#else +typedef unsigned int SecUnsignedInt32; +typedef long long SecInt64; +typedef unsigned long long SecUnsignedInt64; +#endif + +#ifdef SECUREC_FOR_WCHAR +#if defined(SECUREC_VXWORKS_PLATFORM) && !defined(__WINT_TYPE__) +typedef wchar_t wint_t; +#endif +#ifndef WEOF +#define WEOF ((wchar_t)(-1)) +#endif +#define SECUREC_CHAR(x) L ## x +typedef wchar_t SecChar; +typedef wchar_t SecUnsignedChar; +typedef wint_t SecInt; +typedef wint_t SecUnsignedInt; +#else /* no SECUREC_FOR_WCHAR */ +#define SECUREC_CHAR(x) (x) +typedef char SecChar; +typedef unsigned char SecUnsignedChar; +typedef int SecInt; +typedef unsigned int SecUnsignedInt; +#endif +#endif + +/* + * Determine whether the address is 8-byte aligned + * Some systems do not have uintptr_t type, so use NULL to clear tool alarm 507 + */ +#define SECUREC_ADDR_ALIGNED_8(addr) ((((size_t)(addr)) & 7U) == 0) /* Use 7 to check aligned 8 */ + +/* + * If you define the memory allocation function, you need to define the function prototype. + * You can define this macro as a header file. + */ +#if defined(SECUREC_MALLOC_PROTOTYPE) +SECUREC_MALLOC_PROTOTYPE +#endif + +#ifndef SECUREC_MALLOC +#define SECUREC_MALLOC(x) malloc((size_t)(x)) +#endif + +#ifndef SECUREC_FREE +#define SECUREC_FREE(x) free((void *)(x)) +#endif + +/* Improve performance with struct assignment, buf1 is not defined to avoid tool false positive */ +#define SECUREC_COPY_VALUE_BY_STRUCT(dest, src, n) do { \ + *(SecStrBuf##n *)(void *)(dest) = *(const SecStrBuf##n *)(const void *)(src); \ +} SECUREC_WHILE_ZERO + +typedef struct { + unsigned char buf[2]; /* Performance optimization code structure assignment length 2 bytes */ +} SecStrBuf2; +typedef struct { + unsigned char buf[3]; /* Performance optimization code structure assignment length 3 bytes */ +} SecStrBuf3; +typedef struct { + unsigned char buf[4]; /* Performance optimization code structure assignment length 4 bytes */ +} SecStrBuf4; +typedef struct { + unsigned char buf[5]; /* Performance optimization code structure assignment length 5 bytes */ +} SecStrBuf5; +typedef struct { + unsigned char buf[6]; /* Performance optimization code structure assignment length 6 bytes */ +} SecStrBuf6; +typedef struct { + unsigned char buf[7]; /* Performance optimization code structure assignment length 7 bytes */ +} SecStrBuf7; +typedef struct { + unsigned char buf[8]; /* Performance optimization code structure assignment length 8 bytes */ +} SecStrBuf8; +typedef struct { + unsigned char buf[9]; /* Performance optimization code structure assignment length 9 bytes */ +} SecStrBuf9; +typedef struct { + unsigned char buf[10]; /* Performance optimization code structure assignment length 10 bytes */ +} SecStrBuf10; +typedef struct { + unsigned char buf[11]; /* Performance optimization code structure assignment length 11 bytes */ +} SecStrBuf11; +typedef struct { + unsigned char buf[12]; /* Performance optimization code structure assignment length 12 bytes */ +} SecStrBuf12; +typedef struct { + unsigned char buf[13]; /* Performance optimization code structure assignment length 13 bytes */ +} SecStrBuf13; +typedef struct { + unsigned char buf[14]; /* Performance optimization code structure assignment length 14 bytes */ +} SecStrBuf14; +typedef struct { + unsigned char buf[15]; /* Performance optimization code structure assignment length 15 bytes */ +} SecStrBuf15; +typedef struct { + unsigned char buf[16]; /* Performance optimization code structure assignment length 16 bytes */ +} SecStrBuf16; +typedef struct { + unsigned char buf[17]; /* Performance optimization code structure assignment length 17 bytes */ +} SecStrBuf17; +typedef struct { + unsigned char buf[18]; /* Performance optimization code structure assignment length 18 bytes */ +} SecStrBuf18; +typedef struct { + unsigned char buf[19]; /* Performance optimization code structure assignment length 19 bytes */ +} SecStrBuf19; +typedef struct { + unsigned char buf[20]; /* Performance optimization code structure assignment length 20 bytes */ +} SecStrBuf20; +typedef struct { + unsigned char buf[21]; /* Performance optimization code structure assignment length 21 bytes */ +} SecStrBuf21; +typedef struct { + unsigned char buf[22]; /* Performance optimization code structure assignment length 22 bytes */ +} SecStrBuf22; +typedef struct { + unsigned char buf[23]; /* Performance optimization code structure assignment length 23 bytes */ +} SecStrBuf23; +typedef struct { + unsigned char buf[24]; /* Performance optimization code structure assignment length 24 bytes */ +} SecStrBuf24; +typedef struct { + unsigned char buf[25]; /* Performance optimization code structure assignment length 25 bytes */ +} SecStrBuf25; +typedef struct { + unsigned char buf[26]; /* Performance optimization code structure assignment length 26 bytes */ +} SecStrBuf26; +typedef struct { + unsigned char buf[27]; /* Performance optimization code structure assignment length 27 bytes */ +} SecStrBuf27; +typedef struct { + unsigned char buf[28]; /* Performance optimization code structure assignment length 28 bytes */ +} SecStrBuf28; +typedef struct { + unsigned char buf[29]; /* Performance optimization code structure assignment length 29 bytes */ +} SecStrBuf29; +typedef struct { + unsigned char buf[30]; /* Performance optimization code structure assignment length 30 bytes */ +} SecStrBuf30; +typedef struct { + unsigned char buf[31]; /* Performance optimization code structure assignment length 31 bytes */ +} SecStrBuf31; +typedef struct { + unsigned char buf[32]; /* Performance optimization code structure assignment length 32 bytes */ +} SecStrBuf32; +typedef struct { + unsigned char buf[33]; /* Performance optimization code structure assignment length 33 bytes */ +} SecStrBuf33; +typedef struct { + unsigned char buf[34]; /* Performance optimization code structure assignment length 34 bytes */ +} SecStrBuf34; +typedef struct { + unsigned char buf[35]; /* Performance optimization code structure assignment length 35 bytes */ +} SecStrBuf35; +typedef struct { + unsigned char buf[36]; /* Performance optimization code structure assignment length 36 bytes */ +} SecStrBuf36; +typedef struct { + unsigned char buf[37]; /* Performance optimization code structure assignment length 37 bytes */ +} SecStrBuf37; +typedef struct { + unsigned char buf[38]; /* Performance optimization code structure assignment length 38 bytes */ +} SecStrBuf38; +typedef struct { + unsigned char buf[39]; /* Performance optimization code structure assignment length 39 bytes */ +} SecStrBuf39; +typedef struct { + unsigned char buf[40]; /* Performance optimization code structure assignment length 40 bytes */ +} SecStrBuf40; +typedef struct { + unsigned char buf[41]; /* Performance optimization code structure assignment length 41 bytes */ +} SecStrBuf41; +typedef struct { + unsigned char buf[42]; /* Performance optimization code structure assignment length 42 bytes */ +} SecStrBuf42; +typedef struct { + unsigned char buf[43]; /* Performance optimization code structure assignment length 43 bytes */ +} SecStrBuf43; +typedef struct { + unsigned char buf[44]; /* Performance optimization code structure assignment length 44 bytes */ +} SecStrBuf44; +typedef struct { + unsigned char buf[45]; /* Performance optimization code structure assignment length 45 bytes */ +} SecStrBuf45; +typedef struct { + unsigned char buf[46]; /* Performance optimization code structure assignment length 46 bytes */ +} SecStrBuf46; +typedef struct { + unsigned char buf[47]; /* Performance optimization code structure assignment length 47 bytes */ +} SecStrBuf47; +typedef struct { + unsigned char buf[48]; /* Performance optimization code structure assignment length 48 bytes */ +} SecStrBuf48; +typedef struct { + unsigned char buf[49]; /* Performance optimization code structure assignment length 49 bytes */ +} SecStrBuf49; +typedef struct { + unsigned char buf[50]; /* Performance optimization code structure assignment length 50 bytes */ +} SecStrBuf50; +typedef struct { + unsigned char buf[51]; /* Performance optimization code structure assignment length 51 bytes */ +} SecStrBuf51; +typedef struct { + unsigned char buf[52]; /* Performance optimization code structure assignment length 52 bytes */ +} SecStrBuf52; +typedef struct { + unsigned char buf[53]; /* Performance optimization code structure assignment length 53 bytes */ +} SecStrBuf53; +typedef struct { + unsigned char buf[54]; /* Performance optimization code structure assignment length 54 bytes */ +} SecStrBuf54; +typedef struct { + unsigned char buf[55]; /* Performance optimization code structure assignment length 55 bytes */ +} SecStrBuf55; +typedef struct { + unsigned char buf[56]; /* Performance optimization code structure assignment length 56 bytes */ +} SecStrBuf56; +typedef struct { + unsigned char buf[57]; /* Performance optimization code structure assignment length 57 bytes */ +} SecStrBuf57; +typedef struct { + unsigned char buf[58]; /* Performance optimization code structure assignment length 58 bytes */ +} SecStrBuf58; +typedef struct { + unsigned char buf[59]; /* Performance optimization code structure assignment length 59 bytes */ +} SecStrBuf59; +typedef struct { + unsigned char buf[60]; /* Performance optimization code structure assignment length 60 bytes */ +} SecStrBuf60; +typedef struct { + unsigned char buf[61]; /* Performance optimization code structure assignment length 61 bytes */ +} SecStrBuf61; +typedef struct { + unsigned char buf[62]; /* Performance optimization code structure assignment length 62 bytes */ +} SecStrBuf62; +typedef struct { + unsigned char buf[63]; /* Performance optimization code structure assignment length 63 bytes */ +} SecStrBuf63; +typedef struct { + unsigned char buf[64]; /* Performance optimization code structure assignment length 64 bytes */ +} SecStrBuf64; + +/* + * User can change the error handler by modify the following definition, + * such as logging the detail error in file. + */ +#if defined(_DEBUG) || defined(DEBUG) +#if defined(SECUREC_ERROR_HANDLER_BY_ASSERT) +#define SECUREC_ERROR_INVALID_PARAMTER(msg) assert(msg "invalid argument" == NULL) +#define SECUREC_ERROR_INVALID_RANGE(msg) assert(msg "invalid dest buffer size" == NULL) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) assert(msg "buffer overlap" == NULL) +#elif defined(SECUREC_ERROR_HANDLER_BY_PRINTF) +#if SECUREC_IN_KERNEL +#define SECUREC_ERROR_INVALID_PARAMTER(msg) printk("%s invalid argument\n", msg) +#define SECUREC_ERROR_INVALID_RANGE(msg) printk("%s invalid dest buffer size\n", msg) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) printk("%s buffer overlap\n", msg) +#else +#define SECUREC_ERROR_INVALID_PARAMTER(msg) printf("%s invalid argument\n", msg) +#define SECUREC_ERROR_INVALID_RANGE(msg) printf("%s invalid dest buffer size\n", msg) +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) printf("%s buffer overlap\n", msg) +#endif +#elif defined(SECUREC_ERROR_HANDLER_BY_FILE_LOG) +#define SECUREC_ERROR_INVALID_PARAMTER(msg) LogSecureCRuntimeError(msg " EINVAL\n") +#define SECUREC_ERROR_INVALID_RANGE(msg) LogSecureCRuntimeError(msg " ERANGE\n") +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) LogSecureCRuntimeError(msg " EOVERLAP\n") +#endif +#endif + +/* Default handler is none */ +#ifndef SECUREC_ERROR_INVALID_PARAMTER +#define SECUREC_ERROR_INVALID_PARAMTER(msg) +#endif +#ifndef SECUREC_ERROR_INVALID_RANGE +#define SECUREC_ERROR_INVALID_RANGE(msg) +#endif +#ifndef SECUREC_ERROR_BUFFER_OVERLAP +#define SECUREC_ERROR_BUFFER_OVERLAP(msg) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Assembly language memory copy and memory set for X86 or MIPS ... */ +#ifdef SECUREC_USE_ASM +void *memcpy_opt(void *dest, const void *src, size_t n); +void *memset_opt(void *s, int c, size_t n); +#endif + +#if defined(SECUREC_ERROR_HANDLER_BY_FILE_LOG) +void LogSecureCRuntimeError(const char *errDetail); +#endif + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif + diff --git a/dynolog_npu/scripts/apply_dyno_patches.sh b/msmonitor/scripts/apply_dyno_patches.sh similarity index 100% rename from dynolog_npu/scripts/apply_dyno_patches.sh rename to msmonitor/scripts/apply_dyno_patches.sh diff --git a/dynolog_npu/scripts/build.sh b/msmonitor/scripts/build.sh similarity index 88% rename from dynolog_npu/scripts/build.sh rename to msmonitor/scripts/build.sh index aa3508e14faa6bfea06afe0cd3083ad1a5317037..52cd5ad4f133bf5ca95f1a624fd4ad556a6b62f5 100644 --- a/dynolog_npu/scripts/build.sh +++ b/msmonitor/scripts/build.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +export BUILD_PROMETHEUS=1 check_gcc_version() { if ! command -v gcc >/dev/null 2>&1; then @@ -30,8 +31,8 @@ check_rust_version() { local RUST_MAJOR=$(echo $RUST_VERSION | cut -d. -f1) local RUST_MINOR=$(echo $RUST_VERSION | cut -d. -f2) - if [ "$RUST_MAJOR" -lt 1 ] || ([ "$RUST_MAJOR" -eq 1 ] && [ "$RUST_MINOR" -lt 56 ]); then - echo "ERROR: Rust version must be greater than or equal to 1.56.0" + if [ "$RUST_MAJOR" -lt 1 ] || ([ "$RUST_MAJOR" -eq 1 ] && [ "$RUST_MINOR" -lt 81 ]); then + echo "ERROR: Rust version must be greater than or equal to 1.81" echo "Current Rust version: $RUST_VERSION" return 1 fi @@ -98,11 +99,21 @@ if [ -z "$PACKAGE_TYPE" ]; then bash scripts/build.sh echo "Build dynolog success without packaging" elif [ "$PACKAGE_TYPE" = "deb" ]; then + ARCHITECTURE=$(uname -m) + CONTROL_FILE="scripts/debian/control" + ARCH="amd64" + if [[ "$ARCHITECTURE" == "aarch64" ]]; then + sed -i 's/^Architecture: .*/Architecture: arm64/' "$CONTROL_FILE" + ARCH="arm64" + echo "dpkg Architecture set to arm64" + fi + export ARCH=$ARCH bash scripts/debian/make_deb.sh + unset ARCH mv dynolog_*.deb ../../ echo "Build dynolog deb package success" elif [ "$PACKAGE_TYPE" = "rpm" ]; then bash scripts/rpm/make_rpm.sh - mv dynolog_*.rpm ../../ + mv dynolog-*.rpm ../../ echo "Build dynolog rpm package success" fi diff --git a/dynolog_npu/scripts/gen_dyno_patches.sh b/msmonitor/scripts/gen_dyno_patches.sh similarity index 100% rename from dynolog_npu/scripts/gen_dyno_patches.sh rename to msmonitor/scripts/gen_dyno_patches.sh diff --git a/msmonitor/scripts/run_tests.sh b/msmonitor/scripts/run_tests.sh new file mode 100644 index 0000000000000000000000000000000000000000..78329da25d0345bec509165e088301637f4046b6 --- /dev/null +++ b/msmonitor/scripts/run_tests.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# 该脚本用于CI环境，执行系统测试用例 +# Copyright (c) 2025, Huawei Technologies Co., Ltd. +# All rights reserved. + +# 严格模式，任何错误都会导致脚本退出 +set -e + +# 测试目录 +ST_DIR="test/st" + +# 当前目录检查 +if [[ $(basename $(pwd)) != "msmonitor" ]]; then + if [[ -d "msmonitor" ]]; then + echo "进入msmonitor目录" + cd msmonitor + else + echo "错误: 请在msmonitor目录或其父目录下运行此脚本" + exit 1 + fi +fi + +# 设置必要的环境变量 +export LD_LIBRARY_PATH=third_party/dynolog/third_party/prometheus-cpp/_build/lib:$LD_LIBRARY_PATH + +echo "执行系统测试 (test/st 目录)" + +# 检查系统测试目录是否存在 +if [[ ! -d "$ST_DIR" ]]; then + echo "错误: 系统测试目录 $ST_DIR 不存在" + exit 1 +fi + +# 查找所有以test开头的.py文件 +st_files=$(find $ST_DIR -name "test*.py") + +if [[ -z "$st_files" ]]; then + echo "错误: 没有找到测试文件" + exit 1 +fi + +# 执行每个测试文件，遇到失败立即停止 +for test_file in $st_files; do + echo "===============================================" + echo "执行测试: $test_file" + + # 直接执行Python文件 + python "$test_file" + result=$? + + if [ $result -eq 0 ]; then + echo "[通过] 测试成功: $test_file" + else + echo "[失败] 测试失败: $test_file" + echo "===============================================" + echo "测试执行中止: 发现失败的测试" + exit 1 + fi +done + +echo "===============================================" +echo "系统测试执行完毕" +echo "[成功] 所有测试通过" +exit 0 \ No newline at end of file diff --git a/msmonitor/test/st/gen_tls_certs.sh b/msmonitor/test/st/gen_tls_certs.sh new file mode 100644 index 0000000000000000000000000000000000000000..5ae6536beddec7a964ddd3c6b8d01ad1de89fb8c --- /dev/null +++ b/msmonitor/test/st/gen_tls_certs.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -e +set -o pipefail + +# Configuration +CERTS_DIR="certs" +DAYS_VALID=3650 +SERVER_CN="localhost" +SERVER_SAN="DNS:localhost,IP:127.0.0.1" +CLIENT_CN="client" + +# Check if openssl is installed +if ! command -v openssl &> /dev/null; then + echo "Error: openssl is not installed. Please install it first." + exit 1 +fi + +# Create certs directory +mkdir -p "$CERTS_DIR" + +# Generate CA root certificate and private key +echo "Generating CA root certificate and private key..." +openssl genrsa -out "$CERTS_DIR/ca.key" 2048 +openssl req -x509 -new -key "$CERTS_DIR/ca.key" -days "$DAYS_VALID" -out "$CERTS_DIR/ca.crt" -subj "/CN=Test-CA" + +# Generate client private key and CSR +echo "Generating client private key and CSR..." +openssl genrsa -out "$CERTS_DIR/client.key" 2048 +openssl req -new -key "$CERTS_DIR/client.key" -out "$CERTS_DIR/client.csr" -subj "/CN=$CLIENT_CN" + +# Create v3.ext for client certificate +echo "Creating v3.ext for client certificate..." +cat > "$CERTS_DIR/v3.ext" < "$CERTS_DIR/server.ext" <= 3.7 ，tensorboard >= 2.11.2，numpy <= 1.26.3` + +### 2. 安装方式 + +#### 2.1 pip 安装（推荐） + + - 现本插件已经上传到 pypi 社区，用户可在 python 环境下直接通过以下 pip 指令进行安装： + ``` + pip install tb-graph-ascend + ``` + - 也可在 pypi 社区上下载离线 whl 包，传输到无法访问公网的环境上离线安装使用。访问[下载链接](https://pypi.org/project/tb-graph-ascend/#files)选择 whl 包进行下载，之后便可使用指令安装（此处{version}为 whl 包实际版本） + ``` + pip install tb-graph_ascend_{version}-py3-none-any.whl + ``` + +#### 2.2 从源代码安装 + +1. 从仓库下载源码并切换到 master 分支: + + ``` + git clone https://gitee.com/ascend/mstt.git -b master + ``` + +2. 进入目录 `plugins/tensorboard-plugins/tb_graph_ascend` 下 +3. 编译前端代码，根据操作系统选取不同指令 + + ``` + cd fe + // 安装前端依赖 + npm install --force + // Windows系统 + npm run buildWin + // 其他可使用cp指令的系统，如Linux或Mac + npm run buildLinux + ``` + + **注意**: 此步骤需要安装 [Node.js](https://nodejs.org/zh-cn/download) 环境 + +4. 回到上级目录直接安装: + ``` + cd ../ + python setup.py develop + ``` + - 或： 构建 whl 包安装 + ``` + python setup.py bdist_wheel + ``` + 在 `plugins/tensorboard-plugins/tb_graph_ascend/dist` 目录下取出 whl 包，使用以下指令安装（此处{version}为 whl 包实际版本） + ``` + pip install tb-graph_ascend_{version}-py3-none-any.whl + ``` + +### 3. 解析数据说明 + + 将通过[msprobe](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe#10-%E5%88%86%E7%BA%A7%E5%8F%AF%E8%A7%86%E5%8C%96%E6%9E%84%E5%9B%BE%E6%AF%94%E5%AF%B9)工具构图功能采集得到的文件后缀为.vis 的模型结构文件（文件本身为 json 格式）放置于某个文件夹中，路径名称下文称之为 `output_path` + - E.g. \ + `---output_path` \ + `-----output.vis` \ + `-----output2.vis` + +### 4. 启动方式 + +1. 启动 TensorBoard + + ``` + tensorboard --logdir output_path + ``` + + 注意：确保默认端口 6006 可连通。 + + 如果需要切换端口号需要在尾部加上指定的端口号，如`--port=6007` + + ``` + tensorboard --logdir output_path --port=6007 + ``` + +2. 在浏览器上打开 tensorboard + + 在浏览器中打开 URL： `http://localhost:6006`。 + + 注意：如果`--logdir` 指定目录下的文件太大或太多，请等候，刷新浏览器查看加载结果。 + +3. 建议在本地启动tensorboard，如果网络浏览器与启动 TensorBoard 的机器不在同一台机器上，需要远程启动，可参照[远程启动方式](#413-远程查看数据)，但需用户自行评估**安全风险**。 + +## 三、浏览器查看 +**注意：本工具不支持同时通过多个浏览器窗口同时访问同一个 TensorBoard 服务，否则会出现页面无法正常显示的情况。** + +### 3.1 主界面 + + +![输入图片说明](./doc/images/main-page.png) + +### 3.2 操作方式： + +- **节点双击打开，单击选中。** +- **选中的节点边框呈现蓝色，比对场景下若其存在对应节点，则对应节点边框为浅蓝色。** +- **键盘 WS 根据鼠标位置放大缩小，AD 左右移动。** +- **鼠标滚轮上下移动，鼠标可拖动页面。** +- **比对场景鼠标右键可选中节点，并可展开至对应侧的节点并选中。** + +![输入图片说明](./doc/images/operator-image.png) +### 3.3 名称搜索 +![输入图片说明](./doc/images/vis_search_info.png) +### 3.4 精度筛选/溢出筛选 +注意：单图场景不存在精度筛选和溢出筛选，下图为双图比对场景。
+ +![输入图片说明](./doc/images/vis_precision_info.png) +### 3.5 未匹配节点筛选 +参考匹配说明 ，不符合匹配规则的节点为无匹配节点，颜色标灰。适用于排查两个模型结构差异的场景。
+ +![输入图片说明](./doc/images/vis_unmatch_info.png) +### 3.6 手动选择节点匹配 +可通过浏览器界面，通过鼠标选择两个待匹配的灰色节点进行匹配。当前暂不支持真实数据模式。
+点击匹配后会将两个节点的子节点按照Module名称依次匹配，取消匹配后会将子节点的匹配关系清除。
+注意：匹配结束之后，需要点击保存才能持久化到源文件里面 + +![输入图片说明](./doc/images/vis_match_info.png) + +### 3.7 生成匹配配置文件 +可保存已经已匹配节点的匹配关系到配置文件中，并支持读取配置文件中的数据，进行匹配操作。
+默认保存在当前目录下，文件名为`[当前文件名].vis.config`，每次切换文件都会扫描当前录下的后缀名为.vis.config配置文件，并更新配置文件列表。 +注意：匹配结束之后，需要点击保存才能持久化到源文件里面 +![输入图片说明](./doc/images/vis_save_match_info.png) + + + +## 四、附录 + +### 4.1 安全加固建议 + +#### 4.1.1 免责声明 +本工具为基于 TensorBoard 底座开发的插件，使用本插件需要基于 TensorBoard 运行，请自行关注 TensorBoard 相关安全配置和安全风险。 +#### 4.1.2 TensorBoard版本说明 +满足[相关依赖](#1-相关依赖)中要求的 TensorBoard 版本皆可正常使用本插件功能，但为 TensorBoard 本身安全风险考虑，建议使用最新版本 TensorBoard 。 +#### 4.1.3 远程查看数据 + +如果网络浏览器与启动 TensorBoard 的机器不在同一台机器上， TensorBoard 提供了远程查看数据的指令启动方式，但此种方式会将服务器对应端口在局域网内公开，请用户自行关注安全风险。 + + * 在启动指令尾部加上`--bind_all`或`--host={服务器IP}`参数启用远程查看方式，如： + + ``` + tensorboard --logdir output_path --port=6006 --host=xxx.xxx.xxx.xxx + 或 + tensorboard --logdir output_path --port=6006 --bind_all + ``` + + * 在打开浏览器访问界面时，需将 URL 内主机名由`localhost`替换为主机的 ip 地址，如`http://xxx.xxx.xxx.xxx:6006` + +### 4.2 公网地址说明 +[公网地址说明](./doc/公网地址说明.csv) + diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/main-page.png b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/main-page.png new file mode 100644 index 0000000000000000000000000000000000000000..b8e2a6dbcc5f55f3369406148dfc378890ccdc73 Binary files /dev/null and b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/main-page.png differ diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/operator-image.png b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/operator-image.png new file mode 100644 index 0000000000000000000000000000000000000000..b4463c05dc0e6a379d68592ec4129bd397ae0dd6 Binary files /dev/null and b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/operator-image.png differ diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_match_info.png b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_match_info.png new file mode 100644 index 0000000000000000000000000000000000000000..71e7d74991303036f02529c47dee545022647379 Binary files /dev/null and b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_match_info.png differ diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_precision_info.png b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_precision_info.png new file mode 100644 index 0000000000000000000000000000000000000000..79c6ff77f4fffedfcbaee47767d3f8a4f1b0d5b3 Binary files /dev/null and b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_precision_info.png differ diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_save_match_info.png b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_save_match_info.png new file mode 100644 index 0000000000000000000000000000000000000000..3670076eb8dc4cc315cfc89acec3d1d8d739ed6e Binary files /dev/null and b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_save_match_info.png differ diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_search_info.png b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_search_info.png new file mode 100644 index 0000000000000000000000000000000000000000..7c51a804862591005725e1c2e1da0ff0ac152df1 Binary files /dev/null and b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_search_info.png differ diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_unmatch_info.png b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_unmatch_info.png new file mode 100644 index 0000000000000000000000000000000000000000..5f698d109543e6171e2df28bafa83a09d3dd351d Binary files /dev/null and b/plugins/tensorboard-plugins/tb_graph_ascend/doc/images/vis_unmatch_info.png differ diff --git "a/plugins/tensorboard-plugins/tb_graph_ascend/doc/\345\205\254\347\275\221\345\234\260\345\235\200\350\257\264\346\230\216.csv" "b/plugins/tensorboard-plugins/tb_graph_ascend/doc/\345\205\254\347\275\221\345\234\260\345\235\200\350\257\264\346\230\216.csv" new file mode 100644 index 0000000000000000000000000000000000000000..ace10f82da1fc85c01850220d9ec812e9b7ecdce --- /dev/null +++ "b/plugins/tensorboard-plugins/tb_graph_ascend/doc/\345\205\254\347\275\221\345\234\260\345\235\200\350\257\264\346\230\216.csv" @@ -0,0 +1,11 @@ +IPַ/URLַ//ַ,;˵ +http://www.apache.org/licenses/LICENSE-2.0,License +pmail_mindstudio@huawei.com,MindStudioٷ +https://gitee.com/ascend/mstt/tree/master/plugins/tensorboard-plugins/tb_graph_ascend,ֵַ +https://npms.io,npm߹ַ +http://codepen.io/shyndman/pen/,룬ע +https://github.com/webcomponents/shadycss/issues/193,룬ע +http://jsbin.com/temexa/4,룬ע +https://fonts.googleapis.com/,룬ʽļ +https://developer.mozilla.org/,룬ע +https://github.com/vaadin/vaadin-time-picker/issues/145,룬ע diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/.prettierrc b/plugins/tensorboard-plugins/tb_graph_ascend/fe/.prettierrc new file mode 100644 index 0000000000000000000000000000000000000000..e3d2acb00457084b2f6cccafb8c95740e0344485 --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/.prettierrc @@ -0,0 +1,13 @@ +{ + "parser": "typescript", + "semi": true, + "singleQuote": true, + "jsxSingleQuote": false, + "bracketSpacing": true, + "tabWidth": 2, + "useTabs": false, + "trailingComma": "all", + "proseWrap": "always", + "endOfLine": "lf", + "printWidth": 120 +} diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/index.html b/plugins/tensorboard-plugins/tb_graph_ascend/fe/index.html new file mode 100644 index 0000000000000000000000000000000000000000..fbb070b3715607b970b860e2f3f13e6e210581e5 --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/index.html @@ -0,0 +1,30 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/package-lock.json b/plugins/tensorboard-plugins/tb_graph_ascend/fe/package-lock.json new file mode 100644 index 0000000000000000000000000000000000000000..15f1931fe9b6ac4de59e7abf2ef953aac12ab9ed --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/package-lock.json @@ -0,0 +1,6572 @@ +{ + "name": "tb-graph-ascend", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tb-graph-ascend", + "version": "0.1.0", + "dependencies": { + "@polymer/decorators": "^3.0.0", + "@polymer/iron-behaviors": "^3.0.1", + "@polymer/iron-collapse": "^3.0.1", + "@polymer/iron-icon": "^3.0.1", + "@polymer/iron-icons": "^3.0.1", + "@polymer/iron-iconset-svg": "^3.0.1", + "@polymer/iron-list": "^3.1.0", + "@polymer/iron-resizable-behavior": "^3.0.1", + "@polymer/paper-behaviors": "^3.0.1", + "@polymer/paper-button": "^3.0.1", + "@polymer/paper-checkbox": "^3.1.0", + "@polymer/paper-dialog": "^3.0.1", + "@polymer/paper-dropdown-menu": "^3.1.0", + "@polymer/paper-icon-button": "^3.0.2", + "@polymer/paper-item": "^3.0.1", + "@polymer/paper-listbox": "^3.0.1", + "@polymer/paper-progress": "^3.0.1", + "@polymer/paper-tooltip": "^3.0.1", + "@polymer/polymer": "^3.5.1", + "@types/lodash": "^4.17.1", + "@vaadin/button": "24.6.5", + "@vaadin/combo-box": "24.6.5", + "@vaadin/context-menu": "24.6.5", + "@vaadin/details": "24.6.5", + "@vaadin/grid": "24.6.5", + "@vaadin/icon": "24.6.5", + "@vaadin/icons": "24.6.5", + "@vaadin/notification": "24.6.5", + "@vaadin/progress-bar": "24.6.5", + "@vaadin/select": "24.6.5", + "@vaadin/tabs": "24.6.5", + "@vaadin/tabsheet": "24.6.5", + "@vaadin/text-field": "24.6.5", + "@vaadin/tooltip": "24.6.5", + "axios": "^1.8.4", + "clean-webpack-plugin": "^4.0.0", + "cross-env": "^7.0.3", + "css-loader": "^7.1.2", + "d3": "5.7.0", + "dagre": "^0.8.5", + "lodash": "^4.17.21", + "prettier": "^3.4.2", + "style-loader": "^4.0.0" + }, + "devDependencies": { + "@types/d3": "5.7.2", + "@types/lodash": "^4.14.172", + "@types/node": "^16.4.13", + "@types/offscreencanvas": "^2019.6.3", + "@types/requirejs": "^2.1.33", + "@types/resize-observer-browser": "^0.1.6", + "@types/three": "^0.131.0", + "html-loader": "^5.1.0", + "html-webpack-plugin": "^5.6.3", + "inline-chunk-html-plugin": "^1.1.1", + "ts-loader": "^9.5.1", + "tslib": "^2.6.2", + "typescript": "^5.4.5", + "webpack": "^5.96.1", + "webpack-cli": "^5.1.4", + "webpack-dev-server": "4.15.1", + "ws": "8.13.0" + } + }, + "node_modules/@discoveryjs/json-ext": { + "version": "0.5.7", + "resolved": "https://registry.npmmirror.com/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", + "integrity": "sha512-dBVuXR082gk3jsFp7Rd/JI4kytwGHecnCoTtXFb7DB6CNHp4rg5k1bhg0nWdLGLnOV71lmDzGQaLMy8iPLY0pw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.8", + "resolved": "https://registry.npmmirror.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", + "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/source-map": { + "version": "0.3.6", + "resolved": "https://registry.npmmirror.com/@jridgewell/source-map/-/source-map-0.3.6.tgz", + "integrity": "sha512-1ZJTZebgqllO79ue2bm3rIGud/bOe0pP5BjSRCRxxYkEZS8STV7zN84UBbiYu7jy+eCKSnVIUgoWWE/tt+shMQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmmirror.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmmirror.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", + "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@lit-labs/ssr-dom-shim": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/@lit-labs/ssr-dom-shim/-/ssr-dom-shim-1.3.0.tgz", + "integrity": "sha512-nQIWonJ6eFAvUUrSlwyHDm/aE8PBDu5kRpL0vHMg6K8fK3Diq1xdPjTnsJSwxABhaZ+5eBi1btQB5ShUTKo4nQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@lit/reactive-element": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/@lit/reactive-element/-/reactive-element-2.0.4.tgz", + "integrity": "sha512-GFn91inaUa2oHLak8awSIigYz0cU0Payr1rcFsrkf5OJ5eSPxElyZfKh0f2p9FsTiZWXQdWGJeXZICEfXXYSXQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.2.0" + } + }, + "node_modules/@open-wc/dedupe-mixin": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/@open-wc/dedupe-mixin/-/dedupe-mixin-1.4.0.tgz", + "integrity": "sha512-Sj7gKl1TLcDbF7B6KUhtvr+1UCxdhMbNY5KxdU5IfMFWqL8oy1ZeAcCANjoB1TL0AJTcPmcCFsCbHf8X2jGDUA==", + "license": "MIT" + }, + "node_modules/@polymer/decorators": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/@polymer/decorators/-/decorators-3.0.0.tgz", + "integrity": "sha512-qh+VID9nDV9q3ABvIfWgm7/+udl7v2HKsMLPXFm8tj1fI7qr7yWJMFwS3xWBkMmuNPtmkS8MDP0vqLAQIEOWzg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.5" + } + }, + "node_modules/@polymer/font-roboto": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@polymer/font-roboto/-/font-roboto-3.0.2.tgz", + "integrity": "sha512-tx5TauYSmzsIvmSqepUPDYbs4/Ejz2XbZ1IkD7JEGqkdNUJlh+9KU85G56Tfdk/xjEZ8zorFfN09OSwiMrIQWA==", + "license": "BSD-3-Clause" + }, + "node_modules/@polymer/iron-a11y-announcer": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/@polymer/iron-a11y-announcer/-/iron-a11y-announcer-3.2.0.tgz", + "integrity": "sha512-We+hyaFHcg7Ke8ovsoxUpYEXFIJLHxMCDaLehTB4dELS+C+K0zMnGSiqQvb/YzGS+nSYpAfkQIyg1msOCdHMtA==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-a11y-keys-behavior": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-a11y-keys-behavior/-/iron-a11y-keys-behavior-3.0.1.tgz", + "integrity": "sha512-lnrjKq3ysbBPT/74l0Fj0U9H9C35Tpw2C/tpJ8a+5g8Y3YJs1WSZYnEl1yOkw6sEyaxOq/1DkzH0+60gGu5/PQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-autogrow-textarea": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/@polymer/iron-autogrow-textarea/-/iron-autogrow-textarea-3.0.3.tgz", + "integrity": "sha512-5r0VkWrIlm0JIp5E5wlnvkw7slK72lFRZXncmrsLZF+6n1dg2rI8jt7xpFzSmUWrqpcyXwyKaGaDvUjl3j4JLA==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-behaviors": "^3.0.0-pre.26", + "@polymer/iron-flex-layout": "^3.0.0-pre.26", + "@polymer/iron-validatable-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-behaviors": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-behaviors/-/iron-behaviors-3.0.1.tgz", + "integrity": "sha512-IMEwcv1lhf1HSQxuyWOUIL0lOBwmeaoSTpgCJeP9IBYnuB1SPQngmfRuHKgK6/m9LQ9F9miC7p3HeQQUdKAE0w==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-checked-element-behavior": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-checked-element-behavior/-/iron-checked-element-behavior-3.0.1.tgz", + "integrity": "sha512-aDr0cbCNVq49q+pOqa6CZutFh+wWpwPMLpEth9swx+GkAj+gCURhuQkaUYhIo5f2egDbEioR1aeHMnPlU9dQZA==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-form-element-behavior": "^3.0.0-pre.26", + "@polymer/iron-validatable-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-collapse": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-collapse/-/iron-collapse-3.0.1.tgz", + "integrity": "sha512-yg6q5ZyckQR9VL9VmLrSTkSFXWy9AcJC8KtnD5cg0EHRPbakE8I9S/gVAgeP4nMWV2a/BjLLC4IBygcCMDhAGw==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-resizable-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-dropdown": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-dropdown/-/iron-dropdown-3.0.1.tgz", + "integrity": "sha512-22yLhepfcKjuQMfFmRHi/9MPKTqkzgRrmWWW0P5uqK++xle53k2QBO5VYUAYiCN3ZcxIi9lEhZ9YWGeQj2JBig==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-behaviors": "^3.0.0-pre.26", + "@polymer/iron-overlay-behavior": "^3.0.0-pre.27", + "@polymer/neon-animation": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-fit-behavior": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/@polymer/iron-fit-behavior/-/iron-fit-behavior-3.1.0.tgz", + "integrity": "sha512-ABcgIYqrjhmUT8tiuolqeGttF/8pd3sEymUDrO1vXbZu4FWIvoLNndrMDFvs++AGd12Mjf5pYy84NJc6dB8Vig==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-flex-layout": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-flex-layout/-/iron-flex-layout-3.0.1.tgz", + "integrity": "sha512-7gB869czArF+HZcPTVSgvA7tXYFze9EKckvM95NB7SqYF+NnsQyhoXgKnpFwGyo95lUjUW9TFDLUwDXnCYFtkw==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-form-element-behavior": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-form-element-behavior/-/iron-form-element-behavior-3.0.1.tgz", + "integrity": "sha512-G/e2KXyL5AY7mMjmomHkGpgS0uAf4ovNpKhkuUTRnMuMJuf589bKqE85KN4ovE1Tzhv2hJoh/igyD6ekHiYU1A==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-icon": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-icon/-/iron-icon-3.0.1.tgz", + "integrity": "sha512-QLPwirk+UPZNaLnMew9VludXA4CWUCenRewgEcGYwdzVgDPCDbXxy6vRJjmweZobMQv/oVLppT2JZtJFnPxX6g==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-flex-layout": "^3.0.0-pre.26", + "@polymer/iron-meta": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-icons": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-icons/-/iron-icons-3.0.1.tgz", + "integrity": "sha512-xtEI8erH2GIBiF3QxEMyW81XuVjguu6Le5WjEEpX67qd9z7jjmc4T/ke3zRUlnDydex9p8ytcwVpMIKcyvjYAQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-icon": "^3.0.0-pre.26", + "@polymer/iron-iconset-svg": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-iconset-svg": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-iconset-svg/-/iron-iconset-svg-3.0.1.tgz", + "integrity": "sha512-XNwURbNHRw6u2fJe05O5fMYye6GSgDlDqCO+q6K1zAnKIrpgZwf2vTkBd5uCcZwsN0FyCB3mvNZx4jkh85dRDw==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-meta": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-input": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-input/-/iron-input-3.0.1.tgz", + "integrity": "sha512-WLx13kEcbH9GKbj9+pWR6pbJkA5kxn3796ynx6eQd2rueMyUfVTR3GzOvadBKsciUuIuzrxpBWZ2+3UcueVUQQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-announcer": "^3.0.0-pre.26", + "@polymer/iron-validatable-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-list": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/@polymer/iron-list/-/iron-list-3.1.0.tgz", + "integrity": "sha512-Eiv6xd3h3oPmn8SXFntXVfC3ZnegH+KHAxiKLKcOASFSRY3mHnr2AdcnExUJ9ItoCMA5UzKaM/0U22eWzGERtA==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/iron-resizable-behavior": "^3.0.0-pre.26", + "@polymer/iron-scroll-target-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-menu-behavior": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@polymer/iron-menu-behavior/-/iron-menu-behavior-3.0.2.tgz", + "integrity": "sha512-8dpASkFNBIkxAJWsFLWIO1M7tKM0+wKs3PqdeF/dDdBciwoaaFgC2K1XCZFZnbe2t9/nJgemXxVugGZAWpYCGg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/iron-flex-layout": "^3.0.0-pre.26", + "@polymer/iron-selector": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-meta": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-meta/-/iron-meta-3.0.1.tgz", + "integrity": "sha512-pWguPugiLYmWFV9UWxLWzZ6gm4wBwQdDy4VULKwdHCqR7OP7u98h+XDdGZsSlDPv6qoryV/e3tGHlTIT0mbzJA==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-overlay-behavior": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/@polymer/iron-overlay-behavior/-/iron-overlay-behavior-3.0.3.tgz", + "integrity": "sha512-Q/Fp0+uOQQ145ebZ7T8Cxl4m1tUKYjyymkjcL2rXUm+aDQGb1wA1M1LYxUF5YBqd+9lipE0PTIiYwA2ZL/sznA==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/iron-fit-behavior": "^3.0.0-pre.26", + "@polymer/iron-resizable-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-range-behavior": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-range-behavior/-/iron-range-behavior-3.0.1.tgz", + "integrity": "sha512-+jtL9v45M/T1RJleWyQaNH84S9/mIIR+AjNbYIttbKGp1eG+98j8MDWe7LXNtg79V2LQnE/+VS82cBeELyGVeg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-resizable-behavior": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-resizable-behavior/-/iron-resizable-behavior-3.0.1.tgz", + "integrity": "sha512-FyHxRxFspVoRaeZSWpT3y0C9awomb4tXXolIJcZ7RvXhMP632V5lez+ch5G5SwK0LpnAPkg35eB0LPMFv+YMMQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-scroll-target-behavior": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-scroll-target-behavior/-/iron-scroll-target-behavior-3.0.1.tgz", + "integrity": "sha512-xg1WanG25BIkQE8rhuReqY9zx1K5M7F+YAIYpswEp5eyDIaZ1Y3vUmVeQ3KG+hiSugzI1M752azXN7kvyhOBcQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-selector": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-selector/-/iron-selector-3.0.1.tgz", + "integrity": "sha512-sBVk2uas6prW0glUe2xEJJYlvxmYzM40Au9OKbfDK2Qekou/fLKcBRyIYI39kuI8zWRaip8f3CI8qXcUHnKb1A==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/iron-validatable-behavior": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/iron-validatable-behavior/-/iron-validatable-behavior-3.0.1.tgz", + "integrity": "sha512-wwpYh6wOa4fNI+jH5EYKC7TVPYQ2OfgQqocWat7GsNWcsblKYhLYbwsvEY5nO0n2xKqNfZzDLrUom5INJN7msQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-meta": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/neon-animation": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/neon-animation/-/neon-animation-3.0.1.tgz", + "integrity": "sha512-cDDc0llpVCe0ATbDS3clDthI54Bc8YwZIeTGGmBJleKOvbRTUC5+ssJmRL+VwVh+VM5FlnQlx760ppftY3uprg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-resizable-behavior": "^3.0.0-pre.26", + "@polymer/iron-selector": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-behaviors": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-behaviors/-/paper-behaviors-3.0.1.tgz", + "integrity": "sha512-6knhj69fPJejv8qR0kCSUY+Q0XjaUf0OSnkjRjmTJPAwSrRYtgqE+l6P1FfA+py1X/cUjgne9EF5rMZAKJIg1g==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-behaviors": "^3.0.0-pre.26", + "@polymer/iron-checked-element-behavior": "^3.0.0-pre.26", + "@polymer/paper-ripple": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-button": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-button/-/paper-button-3.0.1.tgz", + "integrity": "sha512-JRNBc+Oj9EWnmyLr7FcCr8T1KAnEHPh6mosln9BUdkM+qYaYsudSICh3cjTIbnj6AuF5OJidoLkM1dlyj0j6Zg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-flex-layout": "^3.0.0-pre.26", + "@polymer/paper-behaviors": "^3.0.0-pre.27", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-checkbox": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/@polymer/paper-checkbox/-/paper-checkbox-3.1.0.tgz", + "integrity": "sha512-kXm6yDG1tT8if0XuJ2cc9NF+g8Ev4wG+rnf0a+Sx+O7J6fn1jcnBlYn72FlrfjVjDQZDBFmT6nynhD5PvFw8iQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/iron-checked-element-behavior": "^3.0.0-pre.26", + "@polymer/paper-behaviors": "^3.0.0-pre.27", + "@polymer/paper-ripple": "^3.0.0-pre.26", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-dialog": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-dialog/-/paper-dialog-3.0.1.tgz", + "integrity": "sha512-KvglYbEq7AWJvui2j6WKLnOvgVMeGjovAydGrPRj7kVzCiD49Eq/hpYFJTRV5iDcalWH+mORUpw+jrFnG9+Kgw==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-overlay-behavior": "^3.0.0-pre.27", + "@polymer/neon-animation": "^3.0.0-pre.26", + "@polymer/paper-dialog-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-dialog-behavior": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-dialog-behavior/-/paper-dialog-behavior-3.0.1.tgz", + "integrity": "sha512-wbI4kCK8le/9MHT+IXzvHjoatxf3kd3Yn0tgozAiAwfSZ7N4Ubpi5MHrK0m9S9PeIxKokAgBYdTUrezSE5378A==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-overlay-behavior": "^3.0.0-pre.27", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-dropdown-menu": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/@polymer/paper-dropdown-menu/-/paper-dropdown-menu-3.2.0.tgz", + "integrity": "sha512-2ohwSHF+RLSK6kA0UkkMiMQF6EZcaEYWAA25kfisI6DWie7yozKrpQNsqvwfOEHU6DdDMIotrOtH1TM88YS8Zg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/iron-form-element-behavior": "^3.0.0-pre.26", + "@polymer/iron-icon": "^3.0.0-pre.26", + "@polymer/iron-iconset-svg": "^3.0.0-pre.26", + "@polymer/iron-validatable-behavior": "^3.0.0-pre.26", + "@polymer/paper-behaviors": "^3.0.0-pre.27", + "@polymer/paper-input": "^3.1.0", + "@polymer/paper-menu-button": "^3.1.0", + "@polymer/paper-ripple": "^3.0.0-pre.26", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.3.1" + } + }, + "node_modules/@polymer/paper-icon-button": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@polymer/paper-icon-button/-/paper-icon-button-3.0.2.tgz", + "integrity": "sha512-kOdxQgnKL097bggFF6PWvsBYuWg+MCcoHoTHX6bh/MuZoWFZNjrFntFqwuB4oEbpjCpfm4moA33muPJFj7CihQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-icon": "^3.0.0-pre.26", + "@polymer/paper-behaviors": "^3.0.0-pre.27", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-input": { + "version": "3.2.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-input/-/paper-input-3.2.1.tgz", + "integrity": "sha512-6ghgwQKM6mS0hAQxQqj+tkeEY1VUBqAsrasAm8V5RpNcfSWQC/hhRFxU0beGuKTAhndzezDzWYP6Zz4b8fExGg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/iron-autogrow-textarea": "^3.0.0-pre.26", + "@polymer/iron-behaviors": "^3.0.0-pre.26", + "@polymer/iron-form-element-behavior": "^3.0.0-pre.26", + "@polymer/iron-input": "^3.0.0-pre.26", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-item": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-item/-/paper-item-3.0.1.tgz", + "integrity": "sha512-KTk2N+GsYiI/HuubL3sxebZ6tteQbBOAp4QVLAnbjSPmwl+mJSDWk+omuadesU0bpkCwaWVs3fHuQsmXxy4pkw==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-behaviors": "^3.0.0-pre.26", + "@polymer/iron-flex-layout": "^3.0.0-pre.26", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-listbox": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-listbox/-/paper-listbox-3.0.1.tgz", + "integrity": "sha512-vMLWFpYcggAPmEDBmK+96fFefacOG3GLB1EguTn8+ZkqI+328hNfw1MzHjH68rgCIIUtjmm+9qgB1Sy/MN0a/A==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-behaviors": "^3.0.0-pre.26", + "@polymer/iron-menu-behavior": "^3.0.0-pre.26", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-menu-button": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/@polymer/paper-menu-button/-/paper-menu-button-3.1.0.tgz", + "integrity": "sha512-q0G0/rvYD/FFmIBMGCQWjfXzRqwFw9+WHSYV4uOQzM1Ln8LMXSAd+2CENsbVwtMh6fmBePj15ZlU8SM2dt1WDQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/iron-behaviors": "^3.0.0-pre.26", + "@polymer/iron-dropdown": "^3.0.0-pre.26", + "@polymer/iron-fit-behavior": "^3.1.0", + "@polymer/neon-animation": "^3.0.0-pre.26", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-progress": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-progress/-/paper-progress-3.0.1.tgz", + "integrity": "sha512-5nguG+tmnyoaWKVNG8Smtno2uLSPBgEsT3f20JY8yJTjUBYWaqa8E3l5RLkTRXgA4x9OnvLb8/CdlQWXQIogBg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-flex-layout": "^3.0.0-pre.26", + "@polymer/iron-range-behavior": "^3.0.0-pre.26", + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-ripple": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@polymer/paper-ripple/-/paper-ripple-3.0.2.tgz", + "integrity": "sha512-DnLNvYIMsiayeICroYxx6Q6Hg1cUU8HN2sbutXazlemAlGqdq80qz3TIaVdbpbt/pvjcFGX2HtntMlPstCge8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/iron-a11y-keys-behavior": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-styles": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-styles/-/paper-styles-3.0.1.tgz", + "integrity": "sha512-y6hmObLqlCx602TQiSBKHqjwkE7xmDiFkoxdYGaNjtv4xcysOTdVJsDR/R9UHwIaxJ7gHlthMSykir1nv78++g==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/font-roboto": "^3.0.1", + "@polymer/iron-flex-layout": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/paper-tooltip": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/@polymer/paper-tooltip/-/paper-tooltip-3.0.1.tgz", + "integrity": "sha512-yiUk09opTEnE1lK+tb501ENb+yQBi4p++Ep0eGJAHesVYKVMPNgPphVKkIizkDaU+n0SE+zXfTsRbYyOMDYXSg==", + "license": "BSD-3-Clause", + "dependencies": { + "@polymer/paper-styles": "^3.0.0-pre.26", + "@polymer/polymer": "^3.0.0" + } + }, + "node_modules/@polymer/polymer": { + "version": "3.5.2", + "resolved": "https://registry.npmmirror.com/@polymer/polymer/-/polymer-3.5.2.tgz", + "integrity": "sha512-fWwImY/UH4bb2534DVSaX+Azs2yKg8slkMBHOyGeU2kKx7Xmxp6Lee0jP8p6B3d7c1gFUPB2Z976dTUtX81pQA==", + "license": "BSD-3-Clause", + "dependencies": { + "@webcomponents/shadycss": "^1.9.1" + } + }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmmirror.com/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/bonjour": { + "version": "3.5.13", + "resolved": "https://registry.npmmirror.com/@types/bonjour/-/bonjour-3.5.13.tgz", + "integrity": "sha512-z9fJ5Im06zvUL548KvYNecEVlA7cVDkGUi6kZusb04mpyEFKCIZJvloCcmpmLaIahDpOQGHaHmG6imtPMmPXGQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmmirror.com/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/connect-history-api-fallback": { + "version": "1.5.4", + "resolved": "https://registry.npmmirror.com/@types/connect-history-api-fallback/-/connect-history-api-fallback-1.5.4.tgz", + "integrity": "sha512-n6Cr2xS1h4uAulPRdlw6Jl6s1oG8KrVilPN2yUITEs+K48EzMJJ3W1xy8K5eWuFvjp3R74AOIGSmp2UfBJ8HFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express-serve-static-core": "*", + "@types/node": "*" + } + }, + "node_modules/@types/d3": { + "version": "5.7.2", + "resolved": "https://registry.npmmirror.com/@types/d3/-/d3-5.7.2.tgz", + "integrity": "sha512-7/wClB8ycneWGy3jdvLfXKTd5SoTg9hji7IdJ0RuO9xTY54YpJ8zlcFADcXhY1J3kCBwxp+/1jeN6a5OMwgYOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-array": "^1", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-collection": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-voronoi": "*", + "@types/d3-zoom": "*" + } + }, + "node_modules/@types/d3-array": { + "version": "1.2.12", + "resolved": "https://registry.npmmirror.com/@types/d3-array/-/d3-array-1.2.12.tgz", + "integrity": "sha512-zIq9wCg/JO7MGC6vq3HRDaVYkqgSPIDjpo3JhAQxl7PHYVPA5D9SMiBfjW/ZoAvPd2a+rkovqBg0nS0QOChsJQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-axis": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/@types/d3-axis/-/d3-axis-3.0.6.tgz", + "integrity": "sha512-pYeijfZuBd87T0hGn0FO1vQ/cgLk6E1ALJjfkC0oJ8cbwkZl3TpgS8bVBLZN+2jjGgg38epgxb2zmoGtSfvgMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/@types/d3-brush/-/d3-brush-3.0.6.tgz", + "integrity": "sha512-nH60IZNNxEcrh6L1ZSMNA28rj27ut/2ZmI3r96Zd+1jrZD++zD3LsMIjWlvg4AYrHn/Pqz4CF3veCxGjtbqt7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/@types/d3-chord/-/d3-chord-3.0.6.tgz", + "integrity": "sha512-LFYWWd8nwfwEmTZG9PfQxd17HbNPksHBiJHaKuY1XeqscXacsS2tyoo6OdRsjf+NQYeB6XrNL3a25E3gH69lcg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-collection": { + "version": "1.0.13", + "resolved": "https://registry.npmmirror.com/@types/d3-collection/-/d3-collection-1.0.13.tgz", + "integrity": "sha512-v0Rgw3IZebRyamcwVmtTDCZ8OmQcj4siaYjNc7wGMZT7PmdSHawGsCOQMxyLvZ7lWjfohYLK0oXtilMOMgfY8A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-contour": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/@types/d3-contour/-/d3-contour-3.0.6.tgz", + "integrity": "sha512-BjzLgXGnCWjUSYGfH1cpdo41/hgdWETu4YxpezoztawmqsvCeep+8QGfiY6YbDvfgHz/DkjeIkkZVJavB4a3rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-dispatch": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/@types/d3-dispatch/-/d3-dispatch-3.0.6.tgz", + "integrity": "sha512-4fvZhzMeeuBJYZXRXrRIQnvUYfyXwYmLsdiN7XXmVNQKKw1cM8a5WdID0g1hVFZDqT9ZqZEY5pD44p24VS7iZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-drag": { + "version": "3.0.7", + "resolved": "https://registry.npmmirror.com/@types/d3-drag/-/d3-drag-3.0.7.tgz", + "integrity": "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-dsv": { + "version": "3.0.7", + "resolved": "https://registry.npmmirror.com/@types/d3-dsv/-/d3-dsv-3.0.7.tgz", + "integrity": "sha512-n6QBF9/+XASqcKK6waudgL0pf/S5XHPPI8APyMLLUHd8NqouBGLsU8MgtO7NINGtPBtk9Kko/W4ea0oAspwh9g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-fetch": { + "version": "3.0.7", + "resolved": "https://registry.npmmirror.com/@types/d3-fetch/-/d3-fetch-3.0.7.tgz", + "integrity": "sha512-fTAfNmxSb9SOWNB9IoG5c8Hg6R+AzUHDRlsXsDZsNp6sxAEOP0tkP3gKkNSO/qmHPoBFTxNrjDprVHDQDvo5aA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-dsv": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.10", + "resolved": "https://registry.npmmirror.com/@types/d3-force/-/d3-force-3.0.10.tgz", + "integrity": "sha512-ZYeSaCF3p73RdOKcjj+swRlZfnYpK1EbaDiYICEEp5Q6sUiqFaFQ9qgoshp5CzIyyb/yD09kD9o2zEltCexlgw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-format": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/@types/d3-format/-/d3-format-3.0.4.tgz", + "integrity": "sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-geo": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/@types/d3-geo/-/d3-geo-3.1.0.tgz", + "integrity": "sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.7", + "resolved": "https://registry.npmmirror.com/@types/d3-hierarchy/-/d3-hierarchy-3.1.7.tgz", + "integrity": "sha512-tJFtNoYBtRtkNysX1Xq4sxtjK8YgoWUNpIiUee0/jHGRwqvzYxkq0hGVbbOGSz+JgFxxRu4K8nb3YpG3CMARtg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-polygon": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@types/d3-polygon/-/d3-polygon-3.0.2.tgz", + "integrity": "sha512-ZuWOtMaHCkN9xoeEMr1ubW2nGWsp4nIql+OPQRstu4ypeZ+zk3YKqQT0CXVe/PYqrKpZAi+J9mTs05TKwjXSRA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-quadtree": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/@types/d3-quadtree/-/d3-quadtree-3.0.6.tgz", + "integrity": "sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-random": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/@types/d3-random/-/d3-random-3.0.3.tgz", + "integrity": "sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmmirror.com/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.1.0.tgz", + "integrity": "sha512-iWMJgwkK7yTRmWqRB5plb1kadXyQ5Sj8V/zYlFGMUBbIPKQScw+Dku9cAAMgJG+z5GYDoMjWGLVOvjghDEFnKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-selection": { + "version": "3.0.11", + "resolved": "https://registry.npmmirror.com/@types/d3-selection/-/d3-selection-3.0.11.tgz", + "integrity": "sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-shape": { + "version": "3.1.7", + "resolved": "https://registry.npmmirror.com/@types/d3-shape/-/d3-shape-3.1.7.tgz", + "integrity": "sha512-VLvUQ33C+3J+8p+Daf+nYSOsjB4GXp19/S/aGo60m9h1v6XaxjiT82lKVWJCfzhtuZ3yD7i/TPeC/fuKLLOSmg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-time-format": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/@types/d3-time-format/-/d3-time-format-4.0.3.tgz", + "integrity": "sha512-5xg9rC+wWL8kdDj153qZcsJ0FWiFt0J5RB6LYUNZjwSnesfblqrI/bJ1wBdJ8OQfncgbJG5+2F+qfqnqyzYxyg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-transition": { + "version": "3.0.9", + "resolved": "https://registry.npmmirror.com/@types/d3-transition/-/d3-transition-3.0.9.tgz", + "integrity": "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-voronoi": { + "version": "1.1.12", + "resolved": "https://registry.npmmirror.com/@types/d3-voronoi/-/d3-voronoi-1.1.12.tgz", + "integrity": "sha512-DauBl25PKZZ0WVJr42a6CNvI6efsdzofl9sajqZr2Gf5Gu733WkDdUGiPkUHXiUvYGzNNlFQde2wdZdfQPG+yw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.8", + "resolved": "https://registry.npmmirror.com/@types/d3-zoom/-/d3-zoom-3.0.8.tgz", + "integrity": "sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, + "node_modules/@types/eslint": { + "version": "9.6.1", + "resolved": "https://registry.npmmirror.com/@types/eslint/-/eslint-9.6.1.tgz", + "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==", + "license": "MIT", + "dependencies": { + "@types/estree": "*", + "@types/json-schema": "*" + } + }, + "node_modules/@types/eslint-scope": { + "version": "3.7.7", + "resolved": "https://registry.npmmirror.com/@types/eslint-scope/-/eslint-scope-3.7.7.tgz", + "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==", + "license": "MIT", + "dependencies": { + "@types/eslint": "*", + "@types/estree": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/@types/estree/-/estree-1.0.7.tgz", + "integrity": "sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==", + "license": "MIT" + }, + "node_modules/@types/express": { + "version": "4.17.21", + "resolved": "https://registry.npmmirror.com/@types/express/-/express-4.17.21.tgz", + "integrity": "sha512-ejlPM315qwLpaQlQDTjPdsUFSc6ZsP4AN6AlWnogPjQ7CVi7PYF3YVz+CY3jE2pwYf7E/7HlDAN0rV2GxTG0HQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.0.6", + "resolved": "https://registry.npmmirror.com/@types/express-serve-static-core/-/express-serve-static-core-5.0.6.tgz", + "integrity": "sha512-3xhRnjJPkULekpSzgtoNYYcTWgEZkp4myc+Saevii5JPnHNvHMRlBSHDbs7Bh1iPPoVTERHEZXyhyLbMEsExsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/express/node_modules/@types/express-serve-static-core": { + "version": "4.19.6", + "resolved": "https://registry.npmmirror.com/@types/express-serve-static-core/-/express-serve-static-core-4.19.6.tgz", + "integrity": "sha512-N4LZ2xG7DatVqhCZzOGb1Yi5lMbXSZcmdLDe9EzSndPV2HpWYWzRbaerl2n27irrm94EPpprqa8KpskPT085+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/geojson": { + "version": "7946.0.16", + "resolved": "https://registry.npmmirror.com/@types/geojson/-/geojson-7946.0.16.tgz", + "integrity": "sha512-6C8nqWur3j98U6+lXDfTUWIfgvZU+EumvpHKcYjujKH7woYyLj2sUmff0tRhrqM7BohUw7Pz3ZB1jj2gW9Fvmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/@types/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==", + "license": "MIT", + "dependencies": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "node_modules/@types/html-minifier-terser": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/@types/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", + "integrity": "sha512-oh/6byDPnL1zeNXFrDXFLyZjkr1MsBG667IM792caf1L2UPOOMf65NFzjUH/ltyfwjAGfs1rsX1eftK0jC/KIg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/http-proxy": { + "version": "1.17.16", + "resolved": "https://registry.npmmirror.com/@types/http-proxy/-/http-proxy-1.17.16.tgz", + "integrity": "sha512-sdWoUajOB1cd0A8cRRQ1cfyWNbmFKLAqBB89Y8x5iYyG/mkJHc0YUH8pdWBy2omi9qtCpiIgGjuwO0dQST2l5w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmmirror.com/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/@types/lodash": { + "version": "4.17.16", + "resolved": "https://registry.npmmirror.com/@types/lodash/-/lodash-4.17.16.tgz", + "integrity": "sha512-HX7Em5NYQAXKW+1T+FiuG27NGwzJfCX3s1GjOa7ujxZa52kjJLOr4FUxT+giF6Tgxv1e+/czV/iTtBw27WTU9g==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmmirror.com/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/minimatch": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/@types/minimatch/-/minimatch-5.1.2.tgz", + "integrity": "sha512-K0VQKziLUWkVKiRVrx4a40iPaxTUefQmjtkQofBkYRcoaaL/8rhwDWww9qWbrgicNOgnpIsMxyNIUM4+n6dUIA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "16.18.126", + "resolved": "https://registry.npmmirror.com/@types/node/-/node-16.18.126.tgz", + "integrity": "sha512-OTcgaiwfGFBKacvfwuHzzn1KLxH/er8mluiy8/uM3sGXHaRe73RrSIj01jow9t4kJEW633Ov+cOexXeiApTyAw==", + "license": "MIT" + }, + "node_modules/@types/node-forge": { + "version": "1.3.11", + "resolved": "https://registry.npmmirror.com/@types/node-forge/-/node-forge-1.3.11.tgz", + "integrity": "sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/offscreencanvas": { + "version": "2019.7.3", + "resolved": "https://registry.npmmirror.com/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz", + "integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/qs": { + "version": "6.9.18", + "resolved": "https://registry.npmmirror.com/@types/qs/-/qs-6.9.18.tgz", + "integrity": "sha512-kK7dgTYDyGqS+e2Q4aK9X3D7q234CIZ1Bv0q/7Z5IwRDoADNU81xXJK/YVyLbLTZCoIwUoDoffFeF+p/eIklAA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmmirror.com/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/requirejs": { + "version": "2.1.37", + "resolved": "https://registry.npmmirror.com/@types/requirejs/-/requirejs-2.1.37.tgz", + "integrity": "sha512-jmFgr3mwN2NSmtRP6IpZ2nfRS7ufSXuDYQ6YyPFArN8x5dARQcD/DXzT0J6NYbvquVT4pg9K9HWdi6e6DZR9iQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/resize-observer-browser": { + "version": "0.1.11", + "resolved": "https://registry.npmmirror.com/@types/resize-observer-browser/-/resize-observer-browser-0.1.11.tgz", + "integrity": "sha512-cNw5iH8JkMkb3QkCoe7DaZiawbDQEUX8t7iuQaRTyLOyQCR2h+ibBD4GJt7p5yhUHrlOeL7ZtbxNHeipqNsBzQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmmirror.com/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmmirror.com/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-index": { + "version": "1.9.4", + "resolved": "https://registry.npmmirror.com/@types/serve-index/-/serve-index-1.9.4.tgz", + "integrity": "sha512-qLpGZ/c2fhSs5gnYsQxtDEq3Oy8SXPClIXkW5ghvAvsNuVSA8k+gCONcUCS/UjLEYvYps+e8uBtfgXgvhwfNug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/express": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmmirror.com/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, + "node_modules/@types/sockjs": { + "version": "0.3.36", + "resolved": "https://registry.npmmirror.com/@types/sockjs/-/sockjs-0.3.36.tgz", + "integrity": "sha512-MK9V6NzAS1+Ud7JV9lJLFqW85VbC9dq3LmwZCuBe4wBDgKC0Kj/jd8Xl+nSviU+Qc3+m7umHHyHg//2KSa0a0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/three": { + "version": "0.131.1", + "resolved": "https://registry.npmmirror.com/@types/three/-/three-0.131.1.tgz", + "integrity": "sha512-unnjsolcm7R90e4XK9qMq4JYEzly0XQNa0pG8RAOMZeVzj3FLIFPymAYUx4Osz0gY9jFZz8omIQplqiieEE7gw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/trusted-types": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/@types/trusted-types/-/trusted-types-2.0.7.tgz", + "integrity": "sha512-ScaPdn1dQczgbl0QFTeTOmVHFULt394XJgOQNoyVhZ6r2vLnMLJfBPd53SB52T/3G36VI1/g2MZaX0cwDuXsfw==", + "license": "MIT" + }, + "node_modules/@types/ws": { + "version": "8.18.0", + "resolved": "https://registry.npmmirror.com/@types/ws/-/ws-8.18.0.tgz", + "integrity": "sha512-8svvI3hMyvN0kKCJMvTJP/x6Y/EoQbepff882wL+Sn5QsXb3etnamgrJq4isrBxSJj5L2AuXcI0+bgkoAXGUJw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@vaadin/a11y-base": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/a11y-base/-/a11y-base-24.6.7.tgz", + "integrity": "sha512-CJYYTWPBEEaVt4AvBE8RzEn3hqUZbGUGLzqs6NGBFTw0c5cfkqoO2ZMkKhz5Z52QF+2mCXpEtyg6s+t0h171Qg==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/button": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/button/-/button-24.6.5.tgz", + "integrity": "sha512-i+pgR0Gn6EWxLgWEQOi7yXXQSQklsr7a+yotlet1GOB+DymE+w9RVp4WOZ6T8yaqTICKcDQldFkreTzFVxsHAQ==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/checkbox": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/checkbox/-/checkbox-24.6.7.tgz", + "integrity": "sha512-/Vl5codokNdN5ku1l/iAkdjUmYTUZGKyAleHjM7V3ZFpwkK2IoWN4HrbWyhPuf1gL3T85bKMLSPuYoOX/ymrFw==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.7", + "@vaadin/component-base": "~24.6.7", + "@vaadin/field-base": "~24.6.7", + "@vaadin/vaadin-lumo-styles": "~24.6.7", + "@vaadin/vaadin-material-styles": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/combo-box": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/combo-box/-/combo-box-24.6.5.tgz", + "integrity": "sha512-u/xC9QegwWgmw9TutPRoIzeBpUgG6Kt9CmJbNZNeWBrP9Nicz/QAawApynvjWQtmm7zIKXp7SPzW1Gqwpe09mQ==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/field-base": "~24.6.5", + "@vaadin/input-container": "~24.6.5", + "@vaadin/item": "~24.6.5", + "@vaadin/lit-renderer": "~24.6.5", + "@vaadin/overlay": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/component-base": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/component-base/-/component-base-24.6.7.tgz", + "integrity": "sha512-LcZQZEwouPDHBoXfXRREb1mRScsPSPeKTUZdgrXh180Piy57VzpNzslIMrdfVFSye9lLMs2/g2o8HCUDgnY/OQ==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/vaadin-development-mode-detector": "^2.0.0", + "@vaadin/vaadin-usage-statistics": "^2.1.0", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/context-menu": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/context-menu/-/context-menu-24.6.5.tgz", + "integrity": "sha512-WLFKmoyIG+GI/UQH4EohhBsLhsYPGV1wdE80Gu+0Gl3/aGLm1ofl6ls+iVzC+/AOBIjNFG1TGmrxMIti1zk0PA==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/item": "~24.6.5", + "@vaadin/list-box": "~24.6.5", + "@vaadin/lit-renderer": "~24.6.5", + "@vaadin/overlay": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/details": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/details/-/details-24.6.5.tgz", + "integrity": "sha512-V22OCdRnT7qOVsVpedGfrwDPE9dFWdhFDv66RfkiWGHpPoq0+dYUpP2Y5Iy7YRCxqVnogVBiE8qHPgZAO4U18A==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/button": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/field-base": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/field-base/-/field-base-24.6.7.tgz", + "integrity": "sha512-5MXpAQGZA15/hRdnZrJK5q5Mv8rgOraSyBpC/gjRJ1W1IQ5DrCcb3ltvPATguv0K3vpJwunXGXrGqm/+SGEk0w==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.7", + "@vaadin/component-base": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/grid": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/grid/-/grid-24.6.5.tgz", + "integrity": "sha512-BlZO8+oWTmrnCbZESa73IbMuXfxQu7Viotd88NXY/ixq/8LiQqj2yNHtKTPz2l2QL1ke57ckFsjzN6w52nYc5g==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/checkbox": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/lit-renderer": "~24.6.5", + "@vaadin/text-field": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/icon": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/icon/-/icon-24.6.5.tgz", + "integrity": "sha512-y6Jy69nySb3tZqEIYAYpyGTiNkKS//ro+w6tuD0a0gu+GrfTv90XDNEY9FvGvnUHsM44OoiQRH3kD15kmISkxQ==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/icons": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/icons/-/icons-24.6.5.tgz", + "integrity": "sha512-zd8KKkJ18EI70IQGoCz3hcQed+VFPnqECKci8vt+OJi1n5j7qzPW4sbEOLZxr6cWrnN1eNdSHfJCQWXrFfL0bQ==", + "license": "Apache-2.0", + "dependencies": { + "@polymer/polymer": "^3.0.0", + "@vaadin/icon": "~24.6.5" + } + }, + "node_modules/@vaadin/input-container": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/input-container/-/input-container-24.6.7.tgz", + "integrity": "sha512-376ZyD74jrKvjiM+gE0xNScyZPU7REMBbGXpmM4DpoLYgw60m01D3fliZaOTVDyXc3gvxWIai3L1vCY0KYpD6w==", + "license": "Apache-2.0", + "dependencies": { + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.7", + "@vaadin/vaadin-lumo-styles": "~24.6.7", + "@vaadin/vaadin-material-styles": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/item": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/item/-/item-24.6.7.tgz", + "integrity": "sha512-9xpJEVhgHF3YQGVeet2uakMTH7SyEbQx+uT5Kld/r1CiCYOKUxbERXrFuJ/5/lgakXjDvN1d7rYDcjPb3CUfsQ==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.7", + "@vaadin/component-base": "~24.6.7", + "@vaadin/vaadin-lumo-styles": "~24.6.7", + "@vaadin/vaadin-material-styles": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/list-box": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/list-box/-/list-box-24.6.7.tgz", + "integrity": "sha512-yUBHonI6uD28l2h+CUh2KPzXe+Ptv6UWtNJIIevX/xkQhptquXzE01bVXlh1NcLVppnu21gaxFs/l+/rHlAKpw==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.7", + "@vaadin/component-base": "~24.6.7", + "@vaadin/item": "~24.6.7", + "@vaadin/vaadin-lumo-styles": "~24.6.7", + "@vaadin/vaadin-material-styles": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/lit-renderer": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/lit-renderer/-/lit-renderer-24.6.7.tgz", + "integrity": "sha512-S9daJnGW/X+HBhOriENRYNf8hCFYABmea756onaLS0QoWLkaU3QVPKrhHjZtzNVf/15UcIeAx4C5JlIas2osFA==", + "license": "Apache-2.0", + "dependencies": { + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/notification": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/notification/-/notification-24.6.5.tgz", + "integrity": "sha512-9OgYmZn3qU3pVMaoIRITNs6gymrnswYO7bk9+8e97o3W4A9TIcAO6F2HTgLO5ieMuuOI1DSlVCpXbrM3xBe8pw==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.5", + "@vaadin/lit-renderer": "~24.6.5", + "@vaadin/overlay": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/overlay": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/overlay/-/overlay-24.6.7.tgz", + "integrity": "sha512-3HZ2+Ld/ktOzFt3Ug3EoZeMqX//uKh9rsXd1d3lQl18bwVtSvG81lY7NI6tEQ2dSuniM0yy2tM+mVnV4lZq9Gw==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.7", + "@vaadin/component-base": "~24.6.7", + "@vaadin/vaadin-lumo-styles": "~24.6.7", + "@vaadin/vaadin-material-styles": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/popover": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/popover/-/popover-24.6.7.tgz", + "integrity": "sha512-GqdDsi+x6+6YNBNPC+BvrshrwXlcmL+nR8v5sY+l1TMPVKNWFb2579Qzc9vvu7jMOr2rQd3F+ZjPoMAqgwuZHw==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@vaadin/a11y-base": "~24.6.7", + "@vaadin/component-base": "~24.6.7", + "@vaadin/lit-renderer": "~24.6.7", + "@vaadin/overlay": "~24.6.7", + "@vaadin/vaadin-lumo-styles": "~24.6.7", + "@vaadin/vaadin-material-styles": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/progress-bar": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/progress-bar/-/progress-bar-24.6.5.tgz", + "integrity": "sha512-lJPRV1SAP0Z46pcgQ9RiV8ZVqytDpIDZ7oMJW7WsjS70CAlrqJZF0JoJ3WoqUrHasNhxU7jjx+iXVXw7CzRrDg==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/scroller": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/scroller/-/scroller-24.6.7.tgz", + "integrity": "sha512-JLqrJCVcfo3GELWd8xNLGif+xz4WpiodPn4uW5/kI3lqLKYg7RKhEu9dg1zRpSEUou5SVFQCMB9m+D1AwyoQGQ==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.7", + "@vaadin/component-base": "~24.6.7", + "@vaadin/vaadin-lumo-styles": "~24.6.7", + "@vaadin/vaadin-material-styles": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/select": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/select/-/select-24.6.5.tgz", + "integrity": "sha512-dDVv4d4QLs7EZEJuOkBI/wjmR7mZ5TyUacCKmscq+Ke7DQrq46DuCUjj82+OSFC7z2m3+v5wflfVMciQehR1+Q==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.2.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/button": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/field-base": "~24.6.5", + "@vaadin/input-container": "~24.6.5", + "@vaadin/item": "~24.6.5", + "@vaadin/list-box": "~24.6.5", + "@vaadin/lit-renderer": "~24.6.5", + "@vaadin/overlay": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/tabs": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/tabs/-/tabs-24.6.5.tgz", + "integrity": "sha512-svUqDjwzlnKsAOYB0szST4Tjhspnb007bMf16fhmkM12u3KK053hEZ2TYX7lNVFLC3RiDvGa8i6nCAK2SVXCDQ==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/item": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/tabsheet": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/tabsheet/-/tabsheet-24.6.5.tgz", + "integrity": "sha512-dn4RFFdK+7Hu6Hhq/V0jb1pwwcLxipgMjAmYGsot4vapqFKSdqea1WpVo6TvVkGXCg3TIrYq5SRbzrIzh9FEzg==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.5", + "@vaadin/scroller": "~24.6.5", + "@vaadin/tabs": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/text-field": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/text-field/-/text-field-24.6.5.tgz", + "integrity": "sha512-zujt5k6i6pkVbfUiQlYWBGa/MUAmWeq0xhDLgHIapzUlEIq6gf67KFwEfhfmwdVzGQImFTTKUBWhO4DERRF0Nw==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/field-base": "~24.6.5", + "@vaadin/input-container": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/tooltip": { + "version": "24.6.5", + "resolved": "https://registry.npmmirror.com/@vaadin/tooltip/-/tooltip-24.6.5.tgz", + "integrity": "sha512-IPcMN61PO+u9IgHyM3GCqrzSUQUo13Tysvp58Z7OvtZg/IgQpcEtWkC2m+Qg9rwJAZu/x37Qfd/8on0TQWzlMg==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/a11y-base": "~24.6.5", + "@vaadin/component-base": "~24.6.5", + "@vaadin/overlay": "~24.6.5", + "@vaadin/popover": "~24.6.5", + "@vaadin/vaadin-lumo-styles": "~24.6.5", + "@vaadin/vaadin-material-styles": "~24.6.5", + "@vaadin/vaadin-themable-mixin": "~24.6.5", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/vaadin-development-mode-detector": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/@vaadin/vaadin-development-mode-detector/-/vaadin-development-mode-detector-2.0.7.tgz", + "integrity": "sha512-9FhVhr0ynSR3X2ao+vaIEttcNU5XfzCbxtmYOV8uIRnUCtNgbvMOIcyGBvntsX9I5kvIP2dV3cFAOG9SILJzEA==", + "license": "Apache-2.0" + }, + "node_modules/@vaadin/vaadin-lumo-styles": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/vaadin-lumo-styles/-/vaadin-lumo-styles-24.6.7.tgz", + "integrity": "sha512-DNamU8cVxbaVn3HfRm3pN8ul95xvaem92ByVeEQwdvKaHwLI4m7AdSWKEA+13ST9TdBtCeDW6DjmtGcoEqbqiw==", + "license": "Apache-2.0", + "dependencies": { + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.7", + "@vaadin/icon": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7" + } + }, + "node_modules/@vaadin/vaadin-lumo-styles/node_modules/@vaadin/icon": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/icon/-/icon-24.6.7.tgz", + "integrity": "sha512-+Cv3hLyFSXJAhnuGuPQ+hQcv9/ijZpIprJ6rqWeChvFk+bQOoPgUPx/tj67mOiTcrmV5hYt+dYs4QM7JZ//dGg==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.7", + "@vaadin/vaadin-lumo-styles": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/vaadin-material-styles": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/vaadin-material-styles/-/vaadin-material-styles-24.6.7.tgz", + "integrity": "sha512-7ecHOEZrFEbUz5UVSGapOt/uC7lSYV05RADCNhG16c+WsuN+oxkGIIaThMMCdBcclg5ej/BeTxZlZha8JoNO3g==", + "license": "Apache-2.0", + "dependencies": { + "@polymer/polymer": "^3.0.0", + "@vaadin/component-base": "~24.6.7", + "@vaadin/vaadin-themable-mixin": "~24.6.7" + } + }, + "node_modules/@vaadin/vaadin-themable-mixin": { + "version": "24.6.7", + "resolved": "https://registry.npmmirror.com/@vaadin/vaadin-themable-mixin/-/vaadin-themable-mixin-24.6.7.tgz", + "integrity": "sha512-fiVBvJWInNBq/oXeE0UAQmzadQ7UJE3ns768D1taKOwTMOxiio1UMoUXcVGwni9ASzXrd96S7F6c4aIaVqNx6A==", + "license": "Apache-2.0", + "dependencies": { + "@open-wc/dedupe-mixin": "^1.3.0", + "lit": "^3.0.0" + } + }, + "node_modules/@vaadin/vaadin-usage-statistics": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/@vaadin/vaadin-usage-statistics/-/vaadin-usage-statistics-2.1.3.tgz", + "integrity": "sha512-8r4TNknD7OJQADe3VygeofFR7UNAXZ2/jjBFP5dgI8+2uMfnuGYgbuHivasKr9WSQ64sPej6m8rDoM1uSllXjQ==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@vaadin/vaadin-development-mode-detector": "^2.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/ast/-/ast-1.14.1.tgz", + "integrity": "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/helper-numbers": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.13.2", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.13.2.tgz", + "integrity": "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.13.2", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-api-error/-/helper-api-error-1.13.2.tgz", + "integrity": "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-buffer/-/helper-buffer-1.14.1.tgz", + "integrity": "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-numbers": { + "version": "1.13.2", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-numbers/-/helper-numbers-1.13.2.tgz", + "integrity": "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/floating-point-hex-parser": "1.13.2", + "@webassemblyjs/helper-api-error": "1.13.2", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.13.2", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.13.2.tgz", + "integrity": "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.14.1.tgz", + "integrity": "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/wasm-gen": "1.14.1" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.13.2", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/ieee754/-/ieee754-1.13.2.tgz", + "integrity": "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==", + "license": "MIT", + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.13.2", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/leb128/-/leb128-1.13.2.tgz", + "integrity": "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==", + "license": "Apache-2.0", + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.13.2", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/utf8/-/utf8-1.13.2.tgz", + "integrity": "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==", + "license": "MIT" + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-edit/-/wasm-edit-1.14.1.tgz", + "integrity": "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/helper-wasm-section": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-opt": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1", + "@webassemblyjs/wast-printer": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-gen/-/wasm-gen-1.14.1.tgz", + "integrity": "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-opt/-/wasm-opt-1.14.1.tgz", + "integrity": "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz", + "integrity": "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-api-error": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.14.1", + "resolved": "https://registry.npmmirror.com/@webassemblyjs/wast-printer/-/wast-printer-1.14.1.tgz", + "integrity": "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==", + "license": "MIT", + "dependencies": { + "@webassemblyjs/ast": "1.14.1", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webcomponents/shadycss": { + "version": "1.11.2", + "resolved": "https://registry.npmmirror.com/@webcomponents/shadycss/-/shadycss-1.11.2.tgz", + "integrity": "sha512-vRq+GniJAYSBmTRnhCYPAPq6THYqovJ/gzGThWbgEZUQaBccndGTi1hdiUP15HzEco0I6t4RCtXyX0rsSmwgPw==", + "license": "BSD-3-Clause" + }, + "node_modules/@webpack-cli/configtest": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/@webpack-cli/configtest/-/configtest-2.1.1.tgz", + "integrity": "sha512-wy0mglZpDSiSS0XHrVR+BAdId2+yxPSoJW8fsna3ZpYSlufjvxnP4YbKTCBZnNIcGN4r6ZPXV55X4mYExOfLmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.15.0" + }, + "peerDependencies": { + "webpack": "5.x.x", + "webpack-cli": "5.x.x" + } + }, + "node_modules/@webpack-cli/info": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/@webpack-cli/info/-/info-2.0.2.tgz", + "integrity": "sha512-zLHQdI/Qs1UyT5UBdWNqsARasIA+AaF8t+4u2aS2nEpBQh2mWIVb8qAklq0eUENnC5mOItrIB4LiS9xMtph18A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.15.0" + }, + "peerDependencies": { + "webpack": "5.x.x", + "webpack-cli": "5.x.x" + } + }, + "node_modules/@webpack-cli/serve": { + "version": "2.0.5", + "resolved": "https://registry.npmmirror.com/@webpack-cli/serve/-/serve-2.0.5.tgz", + "integrity": "sha512-lqaoKnRYBdo1UgDX8uF24AfGMifWK19TxPmM5FHc2vAGxrJ/qtyUyFBWoY1tISZdelsQ5fBcOusifo5o5wSJxQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.15.0" + }, + "peerDependencies": { + "webpack": "5.x.x", + "webpack-cli": "5.x.x" + }, + "peerDependenciesMeta": { + "webpack-dev-server": { + "optional": true + } + } + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "license": "BSD-3-Clause" + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmmirror.com/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "license": "Apache-2.0" + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmmirror.com/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/accepts/node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmmirror.com/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/acorn": { + "version": "8.14.1", + "resolved": "https://registry.npmmirror.com/acorn/-/acorn-8.14.1.tgz", + "integrity": "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg==", + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ajv": { + "version": "8.17.1", + "resolved": "https://registry.npmmirror.com/ajv/-/ajv-8.17.1.tgz", + "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "license": "MIT", + "dependencies": { + "ajv": "^8.0.0" + }, + "peerDependencies": { + "ajv": "^8.0.0" + }, + "peerDependenciesMeta": { + "ajv": { + "optional": true + } + } + }, + "node_modules/ajv-keywords": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/ajv-keywords/-/ajv-keywords-5.1.0.tgz", + "integrity": "sha512-YCS/JNFAUyr5vAuhk1DWm1CBxRHW9LbJ2ozWeemrIqpbsqKjHVxYPyi5GC0rjZIT5JxJ3virVTS8wk4i/Z+krw==", + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3" + }, + "peerDependencies": { + "ajv": "^8.8.2" + } + }, + "node_modules/ansi-html-community": { + "version": "0.0.8", + "resolved": "https://registry.npmmirror.com/ansi-html-community/-/ansi-html-community-0.0.8.tgz", + "integrity": "sha512-1APHAyr3+PCamwNw3bXCPp4HFLONZt/yIH0sZp0/469KWNTEy+qN5jQ3GVX6DMZ1UXAi34yVwtTeaG/HpBuuzw==", + "dev": true, + "engines": [ + "node >= 0.8.0" + ], + "license": "Apache-2.0", + "bin": { + "ansi-html": "bin/ansi-html" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha512-Dxr6QJj/RdU/hCaBjOfxW+q6lyuVE6JFWIrAUpuOOhoJJoQ99cUn3igRaHVB5P9WrgFVN0FfArM3x0cueOU8ng==", + "license": "MIT", + "dependencies": { + "array-uniq": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha512-MNha4BWQ6JbwhFhj03YK552f7cb3AzoE8SzeljgChvL1dl3IcvggXVz1DilzySZkCja+CXuZbdW7yATchWn8/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/axios": { + "version": "1.8.4", + "resolved": "https://registry.npmmirror.com/axios/-/axios-1.8.4.tgz", + "integrity": "sha512-eBSYY4Y68NNlHbHBMdeDmKNtDgXWhQsJcGqzO3iLUM0GraQFSS9cVgPX5I9b3lbdFKyYoAEGAZF1DwhTaljNAw==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/batch": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/batch/-/batch-0.6.1.tgz", + "integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==", + "dev": true, + "license": "MIT" + }, + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/body-parser": { + "version": "1.20.3", + "resolved": "https://registry.npmmirror.com/body-parser/-/body-parser-1.20.3.tgz", + "integrity": "sha512-7rAxByjUMqQ3/bHJy7D6OGXvx/MMc4IqBn/X0fcM1QUcAItpZrBEYhWGem+tzXH90c+G01ypMcYJBO9Y30203g==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.13.0", + "raw-body": "2.5.2", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bonjour-service": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/bonjour-service/-/bonjour-service-1.3.0.tgz", + "integrity": "sha512-3YuAUiSkWykd+2Azjgyxei8OWf8thdn8AITIog2M4UICzoqfjlqr64WIjEXZllf/W6vK1goqleSR6brGomxQqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "multicast-dns": "^7.2.5" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "dev": true, + "license": "ISC" + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmmirror.com/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmmirror.com/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.4", + "resolved": "https://registry.npmmirror.com/browserslist/-/browserslist-4.24.4.tgz", + "integrity": "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001688", + "electron-to-chromium": "^1.5.73", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "license": "MIT" + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/camel-case": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/camel-case/-/camel-case-4.1.2.tgz", + "integrity": "sha512-gxGWBrTT1JuMx6R+o5PTXMmUnhnVzLQ9SNutD4YqKtI6ap897t3tKECYla6gCWEkplXnlNybEkZg9GEGxKFCgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "pascal-case": "^3.1.2", + "tslib": "^2.0.3" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001707", + "resolved": "https://registry.npmmirror.com/caniuse-lite/-/caniuse-lite-1.0.30001707.tgz", + "integrity": "sha512-3qtRjw/HQSMlDWf+X79N206fepf4SOOU6SQLMaq/0KkZLmSjPxAkBOQQ+FxbHKfHmYLZFfdWsO3KA90ceHPSnw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmmirror.com/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, + "node_modules/chrome-trace-event": { + "version": "1.0.4", + "resolved": "https://registry.npmmirror.com/chrome-trace-event/-/chrome-trace-event-1.0.4.tgz", + "integrity": "sha512-rNjApaLzuwaOTjCiT8lSDdGN1APCiqkChLMJxJPWLunPAt5fy8xgU9/jNOchV84wfIxrA0lRQB7oCT8jrn/wrQ==", + "license": "MIT", + "engines": { + "node": ">=6.0" + } + }, + "node_modules/clean-css": { + "version": "5.3.3", + "resolved": "https://registry.npmmirror.com/clean-css/-/clean-css-5.3.3.tgz", + "integrity": "sha512-D5J+kHaVb/wKSFcyyV75uCn8fiY4sV38XJoe4CUyGQ+mOU/fMVYUdH1hJC+CJQ5uY3EnW27SbJYS4X8BiLrAFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "source-map": "~0.6.0" + }, + "engines": { + "node": ">= 10.0" + } + }, + "node_modules/clean-webpack-plugin": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/clean-webpack-plugin/-/clean-webpack-plugin-4.0.0.tgz", + "integrity": "sha512-WuWE1nyTNAyW5T7oNyys2EN0cfP2fdRxhxnIQWiAp0bMabPdHhoGxM8A6YL2GhqwgrPnnaemVE7nv5XJ2Fhh2w==", + "license": "MIT", + "dependencies": { + "del": "^4.1.1" + }, + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "webpack": ">=4.0.0 <6.0.0" + } + }, + "node_modules/clone-deep": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/clone-deep/-/clone-deep-4.0.1.tgz", + "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-plain-object": "^2.0.4", + "kind-of": "^6.0.2", + "shallow-clone": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/colorette": { + "version": "2.0.20", + "resolved": "https://registry.npmmirror.com/colorette/-/colorette-2.0.20.tgz", + "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmmirror.com/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmmirror.com/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "license": "MIT" + }, + "node_modules/compressible": { + "version": "2.0.18", + "resolved": "https://registry.npmmirror.com/compressible/-/compressible-2.0.18.tgz", + "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mime-db": ">= 1.43.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/compression": { + "version": "1.8.0", + "resolved": "https://registry.npmmirror.com/compression/-/compression-1.8.0.tgz", + "integrity": "sha512-k6WLKfunuqCYD3t6AsuPGvQWaKwuLLh2/xHNcX4qE+vIfDNXpSqnrhwA7O53R7WVQUnt8dVAIW+YHr7xTgOgGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "compressible": "~2.0.18", + "debug": "2.6.9", + "negotiator": "~0.6.4", + "on-headers": "~1.0.2", + "safe-buffer": "5.2.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmmirror.com/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/connect-history-api-fallback": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/connect-history-api-fallback/-/connect-history-api-fallback-2.0.0.tgz", + "integrity": "sha512-U73+6lQFmfiNPrYbXqr6kZ1i1wiRqXnp2nhMsINseWXO8lDau0LGEffJ8kQi4EjLZympVgRdvqjAgiZ1tgzDDA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmmirror.com/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmmirror.com/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.1", + "resolved": "https://registry.npmmirror.com/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmmirror.com/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmmirror.com/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/css-loader": { + "version": "7.1.2", + "resolved": "https://registry.npmmirror.com/css-loader/-/css-loader-7.1.2.tgz", + "integrity": "sha512-6WvYYn7l/XEGN8Xu2vWFt9nVzrCn39vKyTEFf/ExEyoksJjjSZV/0/35XPlMbpnr6VGhZIUg5yJrL8tGfes/FA==", + "license": "MIT", + "dependencies": { + "icss-utils": "^5.1.0", + "postcss": "^8.4.33", + "postcss-modules-extract-imports": "^3.1.0", + "postcss-modules-local-by-default": "^4.0.5", + "postcss-modules-scope": "^3.2.0", + "postcss-modules-values": "^4.0.0", + "postcss-value-parser": "^4.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "webpack": "^5.27.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/css-select": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/css-select/-/css-select-4.3.0.tgz", + "integrity": "sha512-wPpOYtnsVontu2mODhA19JrqWxNsfdatRKd64kmpRbQgh1KtItko5sTnEpPdpSaJszTOhEMlF/RPz28qj4HqhQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.0.1", + "domhandler": "^4.3.1", + "domutils": "^2.8.0", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "license": "MIT", + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/d3": { + "version": "5.7.0", + "resolved": "https://registry.npmmirror.com/d3/-/d3-5.7.0.tgz", + "integrity": "sha512-8KEIfx+dFm8PlbJN9PI0suazrZ41QcaAufsKE9PRcqYPWLngHIyWJZX96n6IQKePGgeSu0l7rtlueSSNq8Zc3g==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1", + "d3-axis": "1", + "d3-brush": "1", + "d3-chord": "1", + "d3-collection": "1", + "d3-color": "1", + "d3-contour": "1", + "d3-dispatch": "1", + "d3-drag": "1", + "d3-dsv": "1", + "d3-ease": "1", + "d3-fetch": "1", + "d3-force": "1", + "d3-format": "1", + "d3-geo": "1", + "d3-hierarchy": "1", + "d3-interpolate": "1", + "d3-path": "1", + "d3-polygon": "1", + "d3-quadtree": "1", + "d3-random": "1", + "d3-scale": "2", + "d3-scale-chromatic": "1", + "d3-selection": "1", + "d3-shape": "1", + "d3-time": "1", + "d3-time-format": "2", + "d3-timer": "1", + "d3-transition": "1", + "d3-voronoi": "1", + "d3-zoom": "1" + } + }, + "node_modules/d3-array": { + "version": "1.2.4", + "resolved": "https://registry.npmmirror.com/d3-array/-/d3-array-1.2.4.tgz", + "integrity": "sha512-KHW6M86R+FUPYGb3R5XiYjXPq7VzwxZ22buHhAEVG5ztoEcZZMLov530mmccaqA1GghZArjQV46fuc8kUqhhHw==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-axis": { + "version": "1.0.12", + "resolved": "https://registry.npmmirror.com/d3-axis/-/d3-axis-1.0.12.tgz", + "integrity": "sha512-ejINPfPSNdGFKEOAtnBtdkpr24c4d4jsei6Lg98mxf424ivoDP2956/5HDpIAtmHo85lqT4pruy+zEgvRUBqaQ==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-brush": { + "version": "1.1.6", + "resolved": "https://registry.npmmirror.com/d3-brush/-/d3-brush-1.1.6.tgz", + "integrity": "sha512-7RW+w7HfMCPyZLifTz/UnJmI5kdkXtpCbombUSs8xniAyo0vIbrDzDwUJB6eJOgl9u5DQOt2TQlYumxzD1SvYA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-dispatch": "1", + "d3-drag": "1", + "d3-interpolate": "1", + "d3-selection": "1", + "d3-transition": "1" + } + }, + "node_modules/d3-chord": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/d3-chord/-/d3-chord-1.0.6.tgz", + "integrity": "sha512-JXA2Dro1Fxw9rJe33Uv+Ckr5IrAa74TlfDEhE/jfLOaXegMQFQTAgAw9WnZL8+HxVBRXaRGCkrNU7pJeylRIuA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1", + "d3-path": "1" + } + }, + "node_modules/d3-collection": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/d3-collection/-/d3-collection-1.0.7.tgz", + "integrity": "sha512-ii0/r5f4sjKNTfh84Di+DpztYwqKhEyUlKoPrzUFfeSkWxjW49xU2QzO9qrPrNkpdI0XJkfzvmTu8V2Zylln6A==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-color": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/d3-color/-/d3-color-1.4.1.tgz", + "integrity": "sha512-p2sTHSLCJI2QKunbGb7ocOh7DgTAn8IrLx21QRc/BSnodXM4sv6aLQlnfpvehFMLZEfBc6g9pH9SWQccFYfJ9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-contour": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/d3-contour/-/d3-contour-1.3.2.tgz", + "integrity": "sha512-hoPp4K/rJCu0ladiH6zmJUEz6+u3lgR+GSm/QdM2BBvDraU39Vr7YdDCicJcxP1z8i9B/2dJLgDC1NcvlF8WCg==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^1.1.1" + } + }, + "node_modules/d3-dispatch": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/d3-dispatch/-/d3-dispatch-1.0.6.tgz", + "integrity": "sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-drag": { + "version": "1.2.5", + "resolved": "https://registry.npmmirror.com/d3-drag/-/d3-drag-1.2.5.tgz", + "integrity": "sha512-rD1ohlkKQwMZYkQlYVCrSFxsWPzI97+W+PaEIBNTMxRuxz9RF0Hi5nJWHGVJ3Om9d2fRTe1yOBINJyy/ahV95w==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-dispatch": "1", + "d3-selection": "1" + } + }, + "node_modules/d3-dsv": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/d3-dsv/-/d3-dsv-1.2.0.tgz", + "integrity": "sha512-9yVlqvZcSOMhCYzniHE7EVUws7Fa1zgw+/EAV2BxJoG3ME19V6BQFBwI855XQDsxyOuG7NibqRMTtiF/Qup46g==", + "license": "BSD-3-Clause", + "dependencies": { + "commander": "2", + "iconv-lite": "0.4", + "rw": "1" + }, + "bin": { + "csv2json": "bin/dsv2json", + "csv2tsv": "bin/dsv2dsv", + "dsv2dsv": "bin/dsv2dsv", + "dsv2json": "bin/dsv2json", + "json2csv": "bin/json2dsv", + "json2dsv": "bin/json2dsv", + "json2tsv": "bin/json2dsv", + "tsv2csv": "bin/dsv2dsv", + "tsv2json": "bin/dsv2json" + } + }, + "node_modules/d3-ease": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/d3-ease/-/d3-ease-1.0.7.tgz", + "integrity": "sha512-lx14ZPYkhNx0s/2HX5sLFUI3mbasHjSSpwO/KaaNACweVwxUruKyWVcb293wMv1RqTPZyZ8kSZ2NogUZNcLOFQ==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-fetch": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/d3-fetch/-/d3-fetch-1.2.0.tgz", + "integrity": "sha512-yC78NBVcd2zFAyR/HnUiBS7Lf6inSCoWcSxFfw8FYL7ydiqe80SazNwoffcqOfs95XaLo7yebsmQqDKSsXUtvA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-dsv": "1" + } + }, + "node_modules/d3-force": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/d3-force/-/d3-force-1.2.1.tgz", + "integrity": "sha512-HHvehyaiUlVo5CxBJ0yF/xny4xoaxFxDnBXNvNcfW9adORGZfyNF1dj6DGLKyk4Yh3brP/1h3rnDzdIAwL08zg==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-collection": "1", + "d3-dispatch": "1", + "d3-quadtree": "1", + "d3-timer": "1" + } + }, + "node_modules/d3-format": { + "version": "1.4.5", + "resolved": "https://registry.npmmirror.com/d3-format/-/d3-format-1.4.5.tgz", + "integrity": "sha512-J0piedu6Z8iB6TbIGfZgDzfXxUFN3qQRMofy2oPdXzQibYGqPB/9iMcxr/TGalU+2RsyDO+U4f33id8tbnSRMQ==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-geo": { + "version": "1.12.1", + "resolved": "https://registry.npmmirror.com/d3-geo/-/d3-geo-1.12.1.tgz", + "integrity": "sha512-XG4d1c/UJSEX9NfU02KwBL6BYPj8YKHxgBEw5om2ZnTRSbIcego6dhHwcxuSR3clxh0EpE38os1DVPOmnYtTPg==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "1" + } + }, + "node_modules/d3-hierarchy": { + "version": "1.1.9", + "resolved": "https://registry.npmmirror.com/d3-hierarchy/-/d3-hierarchy-1.1.9.tgz", + "integrity": "sha512-j8tPxlqh1srJHAtxfvOUwKNYJkQuBFdM1+JAUfq6xqH5eAqf93L7oG1NVqDa4CpFZNvnNKtCYEUC8KY9yEn9lQ==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-interpolate": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/d3-interpolate/-/d3-interpolate-1.4.0.tgz", + "integrity": "sha512-V9znK0zc3jOPV4VD2zZn0sDhZU3WAE2bmlxdIwwQPPzPjvyLkd8B3JUVdS1IDUFDkWZ72c9qnv1GK2ZagTZ8EA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-color": "1" + } + }, + "node_modules/d3-path": { + "version": "1.0.9", + "resolved": "https://registry.npmmirror.com/d3-path/-/d3-path-1.0.9.tgz", + "integrity": "sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-polygon": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/d3-polygon/-/d3-polygon-1.0.6.tgz", + "integrity": "sha512-k+RF7WvI08PC8reEoXa/w2nSg5AUMTi+peBD9cmFc+0ixHfbs4QmxxkarVal1IkVkgxVuk9JSHhJURHiyHKAuQ==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-quadtree": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/d3-quadtree/-/d3-quadtree-1.0.7.tgz", + "integrity": "sha512-RKPAeXnkC59IDGD0Wu5mANy0Q2V28L+fNe65pOCXVdVuTJS3WPKaJlFHer32Rbh9gIo9qMuJXio8ra4+YmIymA==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-random": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/d3-random/-/d3-random-1.1.2.tgz", + "integrity": "sha512-6AK5BNpIFqP+cx/sreKzNjWbwZQCSUatxq+pPRmFIQaWuoD+NrbVWw7YWpHiXpCQ/NanKdtGDuB+VQcZDaEmYQ==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-scale": { + "version": "2.2.2", + "resolved": "https://registry.npmmirror.com/d3-scale/-/d3-scale-2.2.2.tgz", + "integrity": "sha512-LbeEvGgIb8UMcAa0EATLNX0lelKWGYDQiPdHj+gLblGVhGLyNbaCn3EvrJf0A3Y/uOOU5aD6MTh5ZFCdEwGiCw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-array": "^1.2.0", + "d3-collection": "1", + "d3-format": "1", + "d3-interpolate": "1", + "d3-time": "1", + "d3-time-format": "2" + } + }, + "node_modules/d3-scale-chromatic": { + "version": "1.5.0", + "resolved": "https://registry.npmmirror.com/d3-scale-chromatic/-/d3-scale-chromatic-1.5.0.tgz", + "integrity": "sha512-ACcL46DYImpRFMBcpk9HhtIyC7bTBR4fNOPxwVSl0LfulDAwyiHyPOTqcDG1+t5d4P9W7t/2NAuWu59aKko/cg==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-color": "1", + "d3-interpolate": "1" + } + }, + "node_modules/d3-selection": { + "version": "1.4.2", + "resolved": "https://registry.npmmirror.com/d3-selection/-/d3-selection-1.4.2.tgz", + "integrity": "sha512-SJ0BqYihzOjDnnlfyeHT0e30k0K1+5sR3d5fNueCNeuhZTnGw4M4o8mqJchSwgKMXCNFo+e2VTChiSJ0vYtXkg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-shape": { + "version": "1.3.7", + "resolved": "https://registry.npmmirror.com/d3-shape/-/d3-shape-1.3.7.tgz", + "integrity": "sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-path": "1" + } + }, + "node_modules/d3-time": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/d3-time/-/d3-time-1.1.0.tgz", + "integrity": "sha512-Xh0isrZ5rPYYdqhAVk8VLnMEidhz5aP7htAADH6MfzgmmicPkTo8LhkLxci61/lCB7n7UmE3bN0leRt+qvkLxA==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-time-format": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/d3-time-format/-/d3-time-format-2.3.0.tgz", + "integrity": "sha512-guv6b2H37s2Uq/GefleCDtbe0XZAuy7Wa49VGkPVPMfLL9qObgBST3lEHJBMUp8S7NdLQAGIvr2KXk8Hc98iKQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-time": "1" + } + }, + "node_modules/d3-timer": { + "version": "1.0.10", + "resolved": "https://registry.npmmirror.com/d3-timer/-/d3-timer-1.0.10.tgz", + "integrity": "sha512-B1JDm0XDaQC+uvo4DT79H0XmBskgS3l6Ve+1SBCfxgmtIb1AVrPIoqd+nPSv+loMX8szQ0sVUhGngL7D5QPiXw==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-transition": { + "version": "1.3.2", + "resolved": "https://registry.npmmirror.com/d3-transition/-/d3-transition-1.3.2.tgz", + "integrity": "sha512-sc0gRU4PFqZ47lPVHloMn9tlPcv8jxgOQg+0zjhfZXMQuvppjG6YuwdMBE0TuqCZjeJkLecku/l9R0JPcRhaDA==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-color": "1", + "d3-dispatch": "1", + "d3-ease": "1", + "d3-interpolate": "1", + "d3-selection": "^1.1.0", + "d3-timer": "1" + } + }, + "node_modules/d3-voronoi": { + "version": "1.1.4", + "resolved": "https://registry.npmmirror.com/d3-voronoi/-/d3-voronoi-1.1.4.tgz", + "integrity": "sha512-dArJ32hchFsrQ8uMiTBLq256MpnZjeuBtdHpaDlYuQyjU0CVzCJl/BVW+SkszaAeH95D/8gxqAhgx0ouAWAfRg==", + "license": "BSD-3-Clause" + }, + "node_modules/d3-zoom": { + "version": "1.8.3", + "resolved": "https://registry.npmmirror.com/d3-zoom/-/d3-zoom-1.8.3.tgz", + "integrity": "sha512-VoLXTK4wvy1a0JpH2Il+F2CiOhVu7VRXWF5M/LroMIh3/zBAC3WAt7QoIvPibOavVo20hN6/37vwAsdBejLyKQ==", + "license": "BSD-3-Clause", + "dependencies": { + "d3-dispatch": "1", + "d3-drag": "1", + "d3-interpolate": "1", + "d3-selection": "1", + "d3-transition": "1" + } + }, + "node_modules/dagre": { + "version": "0.8.5", + "resolved": "https://registry.npmmirror.com/dagre/-/dagre-0.8.5.tgz", + "integrity": "sha512-/aTqmnRta7x7MCCpExk7HQL2O4owCT2h8NT//9I1OQ9vt29Pa0BzSAkR5lwFUcQ7491yVi/3CXU9jQ5o0Mn2Sw==", + "license": "MIT", + "dependencies": { + "graphlib": "^2.1.8", + "lodash": "^4.17.15" + } + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmmirror.com/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/default-gateway": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/default-gateway/-/default-gateway-6.0.3.tgz", + "integrity": "sha512-fwSOJsbbNzZ/CUFpqFBqYfYNLj1NbMPm8MMCIzHjC83iSJRBEGmDUxU+WP661BaBQImeC2yHwXtz+P/O9o+XEg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "execa": "^5.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/del": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/del/-/del-4.1.1.tgz", + "integrity": "sha512-QwGuEUouP2kVwQenAsOof5Fv8K9t3D8Ca8NxcXKrIpEHjTXK5J2nXLdP+ALI1cgv8wj7KuwBhTwBkOZSJKM5XQ==", + "license": "MIT", + "dependencies": { + "@types/glob": "^7.1.1", + "globby": "^6.1.0", + "is-path-cwd": "^2.0.0", + "is-path-in-cwd": "^2.0.0", + "p-map": "^2.0.0", + "pify": "^4.0.1", + "rimraf": "^2.6.3" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true, + "license": "MIT" + }, + "node_modules/dns-packet": { + "version": "5.6.1", + "resolved": "https://registry.npmmirror.com/dns-packet/-/dns-packet-5.6.1.tgz", + "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dom-converter": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/dom-converter/-/dom-converter-0.2.0.tgz", + "integrity": "sha512-gd3ypIPfOMr9h5jIKq8E3sHOTCjeirnl0WK5ZdS1AW0Odt0b1PaWaHdJ4Qk4klv+YB9aJBS7mESXjFoDQPu6DA==", + "dev": true, + "license": "MIT", + "dependencies": { + "utila": "~0.4" + } + }, + "node_modules/dom-serializer": { + "version": "1.4.1", + "resolved": "https://registry.npmmirror.com/dom-serializer/-/dom-serializer-1.4.1.tgz", + "integrity": "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag==", + "dev": true, + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.2.0", + "entities": "^2.0.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/dom-serializer/node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "dev": true, + "license": "BSD-2-Clause", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "4.3.1", + "resolved": "https://registry.npmmirror.com/domhandler/-/domhandler-4.3.1.tgz", + "integrity": "sha512-GrwoxYN+uWlzO8uhUXRl0P+kHE4GtVPfYzVLcUxPL7KNdHKj66vvlhiweIHqYYXWlw+T8iLMp42Lm67ghw4WMQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.2.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "2.8.0", + "resolved": "https://registry.npmmirror.com/domutils/-/domutils-2.8.0.tgz", + "integrity": "sha512-w96Cjofp72M5IIhpjgobBimYEfoPjx1Vx0BSX9P30WBdZW2WIKU0T1Bd0kz2eNZ9ikjKgHbEyKx8BB6H1L3h3A==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^1.0.1", + "domelementtype": "^2.2.0", + "domhandler": "^4.2.0" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/dot-case": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/dot-case/-/dot-case-3.0.4.tgz", + "integrity": "sha512-Kv5nKlh6yRrdrGvxeJ2e5y2eRUpkUosIW4A2AS38zwSz27zu7ufDwQPi5Jhs3XAlGNetl3bmnGhQsMtkKJnj3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true, + "license": "MIT" + }, + "node_modules/electron-to-chromium": { + "version": "1.5.126", + "resolved": "https://registry.npmmirror.com/electron-to-chromium/-/electron-to-chromium-1.5.126.tgz", + "integrity": "sha512-AtH1uLcTC72LA4vfYcEJJkrMk/MY/X0ub8Hv7QGAePW2JkeUFHEL/QfS4J77R6M87Sss8O0OcqReSaN1bpyA+Q==", + "license": "ISC" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/enhanced-resolve": { + "version": "5.18.1", + "resolved": "https://registry.npmmirror.com/enhanced-resolve/-/enhanced-resolve-5.18.1.tgz", + "integrity": "sha512-ZSW3ma5GkcQBIpwZTSRAI8N71Uuwgs93IezB7mf7R60tC8ZbJideoDNKjHn2O9KIlx6rkGTTEk1xUCK2E1Y2Yg==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmmirror.com/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/envinfo": { + "version": "7.14.0", + "resolved": "https://registry.npmmirror.com/envinfo/-/envinfo-7.14.0.tgz", + "integrity": "sha512-CO40UI41xDQzhLB1hWyqUKgFhs250pNcGbyGKe1l/e4FSaI/+YE4IMG76GDt0In67WLPACIITC+sOi08x4wIvg==", + "dev": true, + "license": "MIT", + "bin": { + "envinfo": "dist/cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-module-lexer": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/es-module-lexer/-/es-module-lexer-1.6.0.tgz", + "integrity": "sha512-qqnD1yMU6tk/jnaMosogGySTZP8YtUgAffA9nMN+E/rjxcfRQ6IEk7IiozUjgxKoFHBGjTLnrHB/YC45r/59EQ==", + "license": "MIT" + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-set-tostringtag": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true, + "license": "MIT" + }, + "node_modules/eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esrecurse/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmmirror.com/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmmirror.com/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmmirror.com/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dev": true, + "license": "MIT" + }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmmirror.com/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmmirror.com/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/express": { + "version": "4.21.2", + "resolved": "https://registry.npmmirror.com/express/-/express-4.21.2.tgz", + "integrity": "sha512-28HqgMZAmih1Czt9ny7qr6ek2qddF4FclbMzwhCREB6OFfH+rXAnuNCwo1/wFvrtbgsQDb4kSbX9de9lFbrXnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.3", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.7.1", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.3.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.12", + "proxy-addr": "~2.0.7", + "qs": "6.13.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.19.0", + "serve-static": "1.16.2", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmmirror.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "license": "MIT" + }, + "node_modules/fast-uri": { + "version": "3.0.6", + "resolved": "https://registry.npmmirror.com/fast-uri/-/fast-uri-3.0.6.tgz", + "integrity": "sha512-Atfo14OibSv5wAp4VWNsFYE1AchQRTv9cBGWET4pZWHzYshFSS9NQI6I57rdKn9croWVMbYFbLhJ+yJvmZIIHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/fastest-levenshtein": { + "version": "1.0.16", + "resolved": "https://registry.npmmirror.com/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", + "integrity": "sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4.9.1" + } + }, + "node_modules/faye-websocket": { + "version": "0.11.4", + "resolved": "https://registry.npmmirror.com/faye-websocket/-/faye-websocket-0.11.4.tgz", + "integrity": "sha512-CzbClwlXAuiRQAlUyfqPgvPoNKTckTPGfwZV4ZdAhVcP2lh9KUxJg2b5GkE7XbjKQ3YJnQ9z6D9ntLAlB+tP8g==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "websocket-driver": ">=0.5.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmmirror.com/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/finalhandler": { + "version": "1.3.1", + "resolved": "https://registry.npmmirror.com/finalhandler/-/finalhandler-1.3.1.tgz", + "integrity": "sha512-6BN9trH7bp3qvnrRyzsBz+g3lZxTNZTbVO2EV1CS0WIcDbawYVdYvGflME/9QP0h0pYlCDBCTjYa9nZzMDpyxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/flat": { + "version": "5.0.2", + "resolved": "https://registry.npmmirror.com/flat/-/flat-5.0.2.tgz", + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dev": true, + "license": "BSD-3-Clause", + "bin": { + "flat": "cli.js" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmmirror.com/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/form-data": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/form-data/-/form-data-4.0.2.tgz", + "integrity": "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmmirror.com/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmmirror.com/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fs-monkey": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/fs-monkey/-/fs-monkey-1.0.6.tgz", + "integrity": "sha512-b1FMfwetIKymC0eioW7mTywihSQE4oLzQn1dB6rZB5fx/3NpNEdAWeCSMB+60/AeT0TCXsxzAlcYVEFCTAksWg==", + "dev": true, + "license": "Unlicense" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmmirror.com/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmmirror.com/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/glob-to-regexp": { + "version": "0.4.1", + "resolved": "https://registry.npmmirror.com/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", + "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", + "license": "BSD-2-Clause" + }, + "node_modules/globby": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/globby/-/globby-6.1.0.tgz", + "integrity": "sha512-KVbFv2TQtbzCoxAnfD6JcHZTYCzyliEaaeM/gH8qQdkKr5s0OP9scEgvdcngyk7AVdY6YVW/TJHd+lQ/Df3Daw==", + "license": "MIT", + "dependencies": { + "array-union": "^1.0.1", + "glob": "^7.0.3", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/globby/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/pify/-/pify-2.3.0.tgz", + "integrity": "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmmirror.com/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/graphlib": { + "version": "2.1.8", + "resolved": "https://registry.npmmirror.com/graphlib/-/graphlib-2.1.8.tgz", + "integrity": "sha512-jcLLfkpoVGmH7/InMC/1hIvOPSUh38oJtGhvrOFGzioE1DZ+0YW16RgmOJhHiuWTvGiJQ9Z1Ik43JvkRPRvE+A==", + "license": "MIT", + "dependencies": { + "lodash": "^4.17.15" + } + }, + "node_modules/handle-thing": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/handle-thing/-/handle-thing-2.0.1.tgz", + "integrity": "sha512-9Qn4yBxelxoh2Ow62nP+Ka/kMnOXRi8BXnRaUwezLNhqelnN49xKz4F/dPP8OYLxLxq6JDtZb2i9XznUQbNPTg==", + "dev": true, + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-tostringtag": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", + "dependencies": { + "has-symbols": "^1.0.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true, + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/hpack.js": { + "version": "2.1.6", + "resolved": "https://registry.npmmirror.com/hpack.js/-/hpack.js-2.1.6.tgz", + "integrity": "sha512-zJxVehUdMGIKsRaNt7apO2Gqp0BdqW5yaiGHXXmbpvxgBYVZnAql+BJb4RO5ad2MgpbZKn5G6nMnegrH1FcNYQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.1", + "obuf": "^1.0.0", + "readable-stream": "^2.0.1", + "wbuf": "^1.1.0" + } + }, + "node_modules/hpack.js/node_modules/readable-stream": { + "version": "2.3.8", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-2.3.8.tgz", + "integrity": "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==", + "dev": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/hpack.js/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true, + "license": "MIT" + }, + "node_modules/hpack.js/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/html-entities": { + "version": "2.5.3", + "resolved": "https://registry.npmmirror.com/html-entities/-/html-entities-2.5.3.tgz", + "integrity": "sha512-D3AfvN7SjhTgBSA8L1BN4FpPzuEd06uy4lHwSoRWr0lndi9BKaNzPLKGOWZ2ocSGguozr08TTb2jhCLHaemruw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/mdevils" + }, + { + "type": "patreon", + "url": "https://patreon.com/mdevils" + } + ], + "license": "MIT" + }, + "node_modules/html-loader": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/html-loader/-/html-loader-5.1.0.tgz", + "integrity": "sha512-Jb3xwDbsm0W3qlXrCZwcYqYGnYz55hb6aoKQTlzyZPXsPpi6tHXzAfqalecglMQgNvtEfxrCQPaKT90Irt5XDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "html-minifier-terser": "^7.2.0", + "parse5": "^7.1.2" + }, + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + } + }, + "node_modules/html-minifier-terser": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/html-minifier-terser/-/html-minifier-terser-7.2.0.tgz", + "integrity": "sha512-tXgn3QfqPIpGl9o+K5tpcj3/MN4SfLtsx2GWwBC3SSd0tXQGyF3gsSqad8loJgKZGM3ZxbYDd5yhiBIdWpmvLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "camel-case": "^4.1.2", + "clean-css": "~5.3.2", + "commander": "^10.0.0", + "entities": "^4.4.0", + "param-case": "^3.0.4", + "relateurl": "^0.2.7", + "terser": "^5.15.1" + }, + "bin": { + "html-minifier-terser": "cli.js" + }, + "engines": { + "node": "^14.13.1 || >=16.0.0" + } + }, + "node_modules/html-minifier-terser/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmmirror.com/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/html-webpack-plugin": { + "version": "5.6.3", + "resolved": "https://registry.npmmirror.com/html-webpack-plugin/-/html-webpack-plugin-5.6.3.tgz", + "integrity": "sha512-QSf1yjtSAsmf7rYBV7XX86uua4W/vkhIt0xNXKbsi2foEeW7vjJQz4bhnpL3xH+l1ryl1680uNv968Z+X6jSYg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/html-minifier-terser": "^6.0.0", + "html-minifier-terser": "^6.0.2", + "lodash": "^4.17.21", + "pretty-error": "^4.0.0", + "tapable": "^2.0.0" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/html-webpack-plugin" + }, + "peerDependencies": { + "@rspack/core": "0.x || 1.x", + "webpack": "^5.20.0" + }, + "peerDependenciesMeta": { + "@rspack/core": { + "optional": true + }, + "webpack": { + "optional": true + } + } + }, + "node_modules/html-webpack-plugin/node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmmirror.com/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/html-webpack-plugin/node_modules/html-minifier-terser": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/html-minifier-terser/-/html-minifier-terser-6.1.0.tgz", + "integrity": "sha512-YXxSlJBZTP7RS3tWnQw74ooKa6L9b9i9QYXY21eUEvhZ3u9XLfv6OnFsQq6RxkhHygsaUMvYsZRV5rU/OVNZxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "camel-case": "^4.1.2", + "clean-css": "^5.2.2", + "commander": "^8.3.0", + "he": "^1.2.0", + "param-case": "^3.0.4", + "relateurl": "^0.2.7", + "terser": "^5.10.0" + }, + "bin": { + "html-minifier-terser": "cli.js" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/htmlparser2": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/htmlparser2/-/htmlparser2-6.1.0.tgz", + "integrity": "sha512-gyyPk6rgonLFEDGoeRgQNaEUvdJ4ktTmmUh/h2t7s+M8oPpIPxgNACWa+6ESR57kXstwqPiCut0V8NRpcwgU7A==", + "dev": true, + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.0.1", + "domhandler": "^4.0.0", + "domutils": "^2.5.2", + "entities": "^2.0.0" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/entities/-/entities-2.2.0.tgz", + "integrity": "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A==", + "dev": true, + "license": "BSD-2-Clause", + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/http-deceiver": { + "version": "1.2.7", + "resolved": "https://registry.npmmirror.com/http-deceiver/-/http-deceiver-1.2.7.tgz", + "integrity": "sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/http-parser-js": { + "version": "0.5.9", + "resolved": "https://registry.npmmirror.com/http-parser-js/-/http-parser-js-0.5.9.tgz", + "integrity": "sha512-n1XsPy3rXVxlqxVioEWdC+0+M+SQw0DpJynwtOPo1X+ZlvdzTLtDBIJJlDQTnwZIFJrZSzSGmIOUdP8tu+SgLw==", + "dev": true, + "license": "MIT" + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmmirror.com/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-proxy-middleware": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/http-proxy-middleware/-/http-proxy-middleware-2.0.7.tgz", + "integrity": "sha512-fgVY8AV7qU7z/MmXJ/rxwbrtQH4jBQ9m7kp3llF0liB7glmFeVZFBepQb32T3y8n8k2+AEYuMPCpinYW+/CuRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-proxy": "^1.17.8", + "http-proxy": "^1.18.1", + "is-glob": "^4.0.1", + "is-plain-obj": "^3.0.0", + "micromatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "@types/express": "^4.17.13" + }, + "peerDependenciesMeta": { + "@types/express": { + "optional": true + } + } + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmmirror.com/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/icss-utils": { + "version": "5.1.0", + "resolved": "https://registry.npmmirror.com/icss-utils/-/icss-utils-5.1.0.tgz", + "integrity": "sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==", + "license": "ISC", + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmmirror.com/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmmirror.com/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/inline-chunk-html-plugin": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/inline-chunk-html-plugin/-/inline-chunk-html-plugin-1.1.1.tgz", + "integrity": "sha512-6W1eGIj8z/Yla6xJx5il6jJfCxMZS3kVkbiLQThbbjdsDLRIWkUVmpnhfW2l6WAwCW+qfy0zoXVGBZM1E5XF3g==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dev": true + }, + "node_modules/interpret": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/interpret/-/interpret-3.1.1.tgz", + "integrity": "sha512-6xwYfHbajpoF0xLW+iwLkhwgvLoZDfjYfoFNu8ftMoXINzwuymNLd9u/KmwtdT2GbR+/Cz66otEGEVVUHX9QLQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/ipaddr.js": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/ipaddr.js/-/ipaddr.js-2.2.0.tgz", + "integrity": "sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + } + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmmirror.com/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "dev": true, + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmmirror.com/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmmirror.com/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-path-cwd": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-in-cwd": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-path-in-cwd/-/is-path-in-cwd-2.1.0.tgz", + "integrity": "sha512-rNocXHgipO+rvnP6dk3zI20RpOtrAM/kzbB258Uw5BWr3TpXi861yzjo16Dn4hUox07iw5AyeMLHWsujkjzvRQ==", + "license": "MIT", + "dependencies": { + "is-path-inside": "^2.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-inside": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/is-path-inside/-/is-path-inside-2.1.0.tgz", + "integrity": "sha512-wiyhTzfDWsvwAW53OBWF5zuvaOGlZ6PwYxAbPVDhpm+gM09xKQGjBq/8uYN12aDvMxnAnq3dxTyoSoRNmg5YFg==", + "license": "MIT", + "dependencies": { + "path-is-inside": "^1.0.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/is-plain-obj": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/is-plain-obj/-/is-plain-obj-3.0.0.tgz", + "integrity": "sha512-gwsOE28k+23GP1B6vFl1oVh/WOzmawBrKwo5Ev6wMKzPkaXaCDIQKzLnvsA42DRlbVTWorkgTKIviAKCWkfUwA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/jest-worker": { + "version": "27.5.1", + "resolved": "https://registry.npmmirror.com/jest-worker/-/jest-worker-27.5.1.tgz", + "integrity": "sha512-7vuh85V5cdDofPyxn58nrPjBktZo0u9x1g8WtjQol+jZDaE+fhN+cIvTj11GndBnMnyfrUOG1sZQxCdjKh+DKg==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmmirror.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "license": "MIT" + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmmirror.com/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/launch-editor": { + "version": "2.10.0", + "resolved": "https://registry.npmmirror.com/launch-editor/-/launch-editor-2.10.0.tgz", + "integrity": "sha512-D7dBRJo/qcGX9xlvt/6wUYzQxjh5G1RvZPgPv8vi4KRU99DVQL/oW7tnVOCCTm2HGeo3C5HvGE5Yrh6UBoZ0vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picocolors": "^1.0.0", + "shell-quote": "^1.8.1" + } + }, + "node_modules/lit": { + "version": "3.2.1", + "resolved": "https://registry.npmmirror.com/lit/-/lit-3.2.1.tgz", + "integrity": "sha512-1BBa1E/z0O9ye5fZprPtdqnc0BFzxIxTTOO/tQFmyC/hj1O3jL4TfmLBw0WEwjAokdLwpclkvGgDJwTIh0/22w==", + "license": "BSD-3-Clause", + "dependencies": { + "@lit/reactive-element": "^2.0.4", + "lit-element": "^4.1.0", + "lit-html": "^3.2.0" + } + }, + "node_modules/lit-element": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/lit-element/-/lit-element-4.1.1.tgz", + "integrity": "sha512-HO9Tkkh34QkTeUmEdNYhMT8hzLid7YlMlATSi1q4q17HE5d9mrrEHJ/o8O2D0cMi182zK1F3v7x0PWFjrhXFew==", + "license": "BSD-3-Clause", + "dependencies": { + "@lit-labs/ssr-dom-shim": "^1.2.0", + "@lit/reactive-element": "^2.0.4", + "lit-html": "^3.2.0" + } + }, + "node_modules/lit-html": { + "version": "3.2.1", + "resolved": "https://registry.npmmirror.com/lit-html/-/lit-html-3.2.1.tgz", + "integrity": "sha512-qI/3lziaPMSKsrwlxH/xMgikhQ0EGOX2ICU73Bi/YHFvz2j/yMCIrw4+puF2IpQ4+upd3EWbvnHM9+PnJn48YA==", + "license": "BSD-3-Clause", + "dependencies": { + "@types/trusted-types": "^2.0.2" + } + }, + "node_modules/loader-runner": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/loader-runner/-/loader-runner-4.3.0.tgz", + "integrity": "sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==", + "license": "MIT", + "engines": { + "node": ">=6.11.5" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmmirror.com/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "license": "MIT" + }, + "node_modules/lower-case": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/lower-case/-/lower-case-2.0.2.tgz", + "integrity": "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "tslib": "^2.0.3" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmmirror.com/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/memfs": { + "version": "3.5.3", + "resolved": "https://registry.npmmirror.com/memfs/-/memfs-3.5.3.tgz", + "integrity": "sha512-UERzLsxzllchadvbPs5aolHh65ISpKpM+ccLbOJ8/vvpBKmAWf+la7dXFy7Mr0ySHbdHrFv5kGFCUHHe6GFEmw==", + "dev": true, + "license": "Unlicense", + "dependencies": { + "fs-monkey": "^1.0.4" + }, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmmirror.com/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmmirror.com/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmmirror.com/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmmirror.com/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmmirror.com/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "dev": true, + "license": "ISC" + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true, + "license": "MIT" + }, + "node_modules/multicast-dns": { + "version": "7.2.5", + "resolved": "https://registry.npmmirror.com/multicast-dns/-/multicast-dns-7.2.5.tgz", + "integrity": "sha512-2eznPJP8z2BFLX50tf0LuODrpINqP1RVIm/CObbTcBRITQgmC/TjcREF1NeTBzIcR5XO/ukWo+YHOjBbFwIupg==", + "dev": true, + "license": "MIT", + "dependencies": { + "dns-packet": "^5.2.2", + "thunky": "^1.0.2" + }, + "bin": { + "multicast-dns": "cli.js" + } + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmmirror.com/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/negotiator": { + "version": "0.6.4", + "resolved": "https://registry.npmmirror.com/negotiator/-/negotiator-0.6.4.tgz", + "integrity": "sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmmirror.com/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "license": "MIT" + }, + "node_modules/no-case": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/no-case/-/no-case-3.0.4.tgz", + "integrity": "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "lower-case": "^2.0.2", + "tslib": "^2.0.3" + } + }, + "node_modules/node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmmirror.com/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "dev": true, + "license": "(BSD-3-Clause OR GPL-2.0)", + "engines": { + "node": ">= 6.13.0" + } + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmmirror.com/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmmirror.com/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmmirror.com/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmmirror.com/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/obuf": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/obuf/-/obuf-1.1.2.tgz", + "integrity": "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==", + "dev": true, + "license": "MIT" + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmmirror.com/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/on-headers": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/on-headers/-/on-headers-1.0.2.tgz", + "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmmirror.com/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmmirror.com/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmmirror.com/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmmirror.com/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-map": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/p-map/-/p-map-2.1.0.tgz", + "integrity": "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmmirror.com/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmmirror.com/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/param-case": { + "version": "3.0.4", + "resolved": "https://registry.npmmirror.com/param-case/-/param-case-3.0.4.tgz", + "integrity": "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==", + "dev": true, + "license": "MIT", + "dependencies": { + "dot-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/parse5": { + "version": "7.2.1", + "resolved": "https://registry.npmmirror.com/parse5/-/parse5-7.2.1.tgz", + "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "entities": "^4.5.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/pascal-case": { + "version": "3.1.2", + "resolved": "https://registry.npmmirror.com/pascal-case/-/pascal-case-3.1.2.tgz", + "integrity": "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g==", + "dev": true, + "license": "MIT", + "dependencies": { + "no-case": "^3.0.4", + "tslib": "^2.0.3" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha512-DUWJr3+ULp4zXmol/SZkFf3JGsS9/SIv+Y3Rt93/UjPpDpklB5f1er4O3POIbUuUJ3FXgqte2Q7SrU6zAqwk8w==", + "license": "(WTFPL OR MIT)" + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmmirror.com/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmmirror.com/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmmirror.com/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmmirror.com/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmmirror.com/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmmirror.com/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmmirror.com/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha512-0Gni6D4UcLTbv9c57DfxDGdr41XfgUjqWZu492f0cIGr16zDU06BWP/RAEvOuo7CQ0CNjHaLlM59YJJFm3NWlw==", + "license": "MIT", + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/postcss": { + "version": "8.5.3", + "resolved": "https://registry.npmmirror.com/postcss/-/postcss-8.5.3.tgz", + "integrity": "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.8", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-modules-extract-imports": { + "version": "3.1.0", + "resolved": "https://registry.npmmirror.com/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.1.0.tgz", + "integrity": "sha512-k3kNe0aNFQDAZGbin48pL2VNidTF0w4/eASDsxlyspobzU3wZQLOGj7L9gfRe0Jo9/4uud09DsjFNH7winGv8Q==", + "license": "ISC", + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-local-by-default": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.2.0.tgz", + "integrity": "sha512-5kcJm/zk+GJDSfw+V/42fJ5fhjL5YbFDl8nVdXkJPLLW+Vf9mTD5Xe0wqIaDnLuL2U6cDNpTr+UQ+v2HWIBhzw==", + "license": "MIT", + "dependencies": { + "icss-utils": "^5.0.0", + "postcss-selector-parser": "^7.0.0", + "postcss-value-parser": "^4.1.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-scope": { + "version": "3.2.1", + "resolved": "https://registry.npmmirror.com/postcss-modules-scope/-/postcss-modules-scope-3.2.1.tgz", + "integrity": "sha512-m9jZstCVaqGjTAuny8MdgE88scJnCiQSlSrOWcTQgM2t32UBe+MUmFSO5t7VMSfAf/FJKImAxBav8ooCHJXCJA==", + "license": "ISC", + "dependencies": { + "postcss-selector-parser": "^7.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-values": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz", + "integrity": "sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==", + "license": "ISC", + "dependencies": { + "icss-utils": "^5.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-selector-parser": { + "version": "7.1.0", + "resolved": "https://registry.npmmirror.com/postcss-selector-parser/-/postcss-selector-parser-7.1.0.tgz", + "integrity": "sha512-8sLjZwK0R+JlxlYcTuVnyT2v+htpdrjDOKuMcOVdYjt52Lh8hWRYpxBPoKx/Zg+bcjc3wx6fmQevMmUztS/ccA==", + "license": "MIT", + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmmirror.com/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "license": "MIT" + }, + "node_modules/prettier": { + "version": "3.5.3", + "resolved": "https://registry.npmmirror.com/prettier/-/prettier-3.5.3.tgz", + "integrity": "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/pretty-error": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/pretty-error/-/pretty-error-4.0.0.tgz", + "integrity": "sha512-AoJ5YMAcXKYxKhuJGdcvse+Voc6v1RgnsR3nWcYU7q4t6z0Q6T86sv5Zq8VIRbOWWFpvdGE83LtdSMNd+6Y0xw==", + "dev": true, + "license": "MIT", + "dependencies": { + "lodash": "^4.17.20", + "renderkid": "^3.0.0" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmmirror.com/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-addr/node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/qs": { + "version": "6.13.0", + "resolved": "https://registry.npmmirror.com/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.0.6" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmmirror.com/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.2", + "resolved": "https://registry.npmmirror.com/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", + "dev": true, + "license": "MIT", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmmirror.com/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmmirror.com/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/rechoir": { + "version": "0.8.0", + "resolved": "https://registry.npmmirror.com/rechoir/-/rechoir-0.8.0.tgz", + "integrity": "sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve": "^1.20.0" + }, + "engines": { + "node": ">= 10.13.0" + } + }, + "node_modules/relateurl": { + "version": "0.2.7", + "resolved": "https://registry.npmmirror.com/relateurl/-/relateurl-0.2.7.tgz", + "integrity": "sha512-G08Dxvm4iDN3MLM0EsP62EDV9IuhXPR6blNz6Utcp7zyV3tr4HVNINt6MpaRWbxoOHT3Q7YN2P+jaHX8vUbgog==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/renderkid": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/renderkid/-/renderkid-3.0.0.tgz", + "integrity": "sha512-q/7VIQA8lmM1hF+jn+sFSPWGlMkSAeNYcPLmDQx2zzuiDfaLrOmumR8iaUKlenFgh0XRPIUeSPlH3A+AW3Z5pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "css-select": "^4.1.3", + "dom-converter": "^0.2.0", + "htmlparser2": "^6.1.0", + "lodash": "^4.17.21", + "strip-ansi": "^6.0.1" + } + }, + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmmirror.com/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmmirror.com/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmmirror.com/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/rw": { + "version": "1.3.3", + "resolved": "https://registry.npmmirror.com/rw/-/rw-1.3.3.tgz", + "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==", + "license": "BSD-3-Clause" + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmmirror.com/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmmirror.com/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/schema-utils": { + "version": "4.3.0", + "resolved": "https://registry.npmmirror.com/schema-utils/-/schema-utils-4.3.0.tgz", + "integrity": "sha512-Gf9qqc58SpCA/xdziiHz35F4GNIWYWZrEshUc/G/r5BnLph6xpKuLeoJoQuj5WfBIx/eQLf+hmVPYHaxJu7V2g==", + "license": "MIT", + "dependencies": { + "@types/json-schema": "^7.0.9", + "ajv": "^8.9.0", + "ajv-formats": "^2.1.1", + "ajv-keywords": "^5.1.0" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + } + }, + "node_modules/select-hose": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/select-hose/-/select-hose-2.0.0.tgz", + "integrity": "sha512-mEugaLK+YfkijB4fx0e6kImuJdCIt2LxCRcbEYPqRGCs4F2ogyfZU5IAZRdjCP8JPq2AtdNoC/Dux63d9Kiryg==", + "dev": true, + "license": "MIT" + }, + "node_modules/selfsigned": { + "version": "2.4.1", + "resolved": "https://registry.npmmirror.com/selfsigned/-/selfsigned-2.4.1.tgz", + "integrity": "sha512-th5B4L2U+eGLq1TVh7zNRGBapioSORUeymIydxgFpwww9d2qyKvtuPU2jJuHvYAwwqi2Y596QBL3eEqcPEYL8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node-forge": "^1.3.0", + "node-forge": "^1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmmirror.com/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/send": { + "version": "0.19.0", + "resolved": "https://registry.npmmirror.com/send/-/send-0.19.0.tgz", + "integrity": "sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/serialize-javascript": { + "version": "6.0.2", + "resolved": "https://registry.npmmirror.com/serialize-javascript/-/serialize-javascript-6.0.2.tgz", + "integrity": "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g==", + "license": "BSD-3-Clause", + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/serve-index": { + "version": "1.9.1", + "resolved": "https://registry.npmmirror.com/serve-index/-/serve-index-1.9.1.tgz", + "integrity": "sha512-pXHfKNP4qujrtteMrSBb0rc8HJ9Ms/GrXwcUtUtD5s4ewDJI8bT3Cz2zTVRMKtri49pLx2e0Ya8ziP5Ya2pZZw==", + "dev": true, + "license": "MIT", + "dependencies": { + "accepts": "~1.3.4", + "batch": "0.6.1", + "debug": "2.6.9", + "escape-html": "~1.0.3", + "http-errors": "~1.6.2", + "mime-types": "~2.1.17", + "parseurl": "~1.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/serve-index/node_modules/depd": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/depd/-/depd-1.1.2.tgz", + "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/http-errors": { + "version": "1.6.3", + "resolved": "https://registry.npmmirror.com/http-errors/-/http-errors-1.6.3.tgz", + "integrity": "sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "depd": "~1.1.2", + "inherits": "2.0.3", + "setprototypeof": "1.1.0", + "statuses": ">= 1.4.0 < 2" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-index/node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmmirror.com/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", + "dev": true, + "license": "ISC" + }, + "node_modules/serve-index/node_modules/setprototypeof": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/setprototypeof/-/setprototypeof-1.1.0.tgz", + "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/serve-index/node_modules/statuses": { + "version": "1.5.0", + "resolved": "https://registry.npmmirror.com/statuses/-/statuses-1.5.0.tgz", + "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/serve-static": { + "version": "1.16.2", + "resolved": "https://registry.npmmirror.com/serve-static/-/serve-static-1.16.2.tgz", + "integrity": "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==", + "dev": true, + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.19.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmmirror.com/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true, + "license": "ISC" + }, + "node_modules/shallow-clone": { + "version": "3.0.1", + "resolved": "https://registry.npmmirror.com/shallow-clone/-/shallow-clone-3.0.1.tgz", + "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==", + "dev": true, + "license": "MIT", + "dependencies": { + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/shell-quote": { + "version": "1.8.2", + "resolved": "https://registry.npmmirror.com/shell-quote/-/shell-quote-1.8.2.tgz", + "integrity": "sha512-AzqKpGKjrj7EM6rKVQEPpB288oCfnrEIuyoT9cyF4nmGa7V8Zk6f7RRqYisX8X9m+Q7bd632aZW4ky7EhbQztA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "dev": true, + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmmirror.com/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/sockjs": { + "version": "0.3.24", + "resolved": "https://registry.npmmirror.com/sockjs/-/sockjs-0.3.24.tgz", + "integrity": "sha512-GJgLTZ7vYb/JtPSSZ10hsOYIvEYsjbNU+zPdIHcUaWVNUEPivzxku31865sSSud0Da0W4lEeOPlmw93zLQchuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "faye-websocket": "^0.11.3", + "uuid": "^8.3.2", + "websocket-driver": "^0.7.4" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmmirror.com/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.21", + "resolved": "https://registry.npmmirror.com/source-map-support/-/source-map-support-0.5.21.tgz", + "integrity": "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==", + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/spdy": { + "version": "4.0.2", + "resolved": "https://registry.npmmirror.com/spdy/-/spdy-4.0.2.tgz", + "integrity": "sha512-r46gZQZQV+Kl9oItvl1JZZqJKGr+oEkB08A6BzkiR7593/7IbtuncXHd2YoYeTsG4157ZssMu9KYvUHLcjcDoA==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.0", + "handle-thing": "^2.0.0", + "http-deceiver": "^1.2.7", + "select-hose": "^2.0.0", + "spdy-transport": "^3.0.0" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/spdy-transport": { + "version": "3.0.0", + "resolved": "https://registry.npmmirror.com/spdy-transport/-/spdy-transport-3.0.0.tgz", + "integrity": "sha512-hsLVFE5SjA6TCisWeJXFKniGGOpBgMLmerfO2aCyCU5s7nJ/rpAepqmFifv/GCbSbueEeAJJnmSQ2rKC/g8Fcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "debug": "^4.1.0", + "detect-node": "^2.0.4", + "hpack.js": "^2.1.6", + "obuf": "^1.1.2", + "readable-stream": "^3.0.6", + "wbuf": "^1.7.3" + } + }, + "node_modules/spdy-transport/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/spdy-transport/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/spdy/node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmmirror.com/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/spdy/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmmirror.com/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmmirror.com/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmmirror.com/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmmirror.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/style-loader": { + "version": "4.0.0", + "resolved": "https://registry.npmmirror.com/style-loader/-/style-loader-4.0.0.tgz", + "integrity": "sha512-1V4WqhhZZgjVAVJyt7TdDPZoPBPNHbekX4fWnCJL1yQukhCeZhJySUL+gL9y6sNdN95uEOS83Y55SqHcP7MzLA==", + "license": "MIT", + "engines": { + "node": ">= 18.12.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.27.0" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmmirror.com/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmmirror.com/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/terser": { + "version": "5.39.0", + "resolved": "https://registry.npmmirror.com/terser/-/terser-5.39.0.tgz", + "integrity": "sha512-LBAhFyLho16harJoWMg/nZsQYgTrg5jXOn2nCYjRUcZZEdE3qa2zb8QEDRUGVZBW4rlazf2fxkg8tztybTaqWw==", + "license": "BSD-2-Clause", + "dependencies": { + "@jridgewell/source-map": "^0.3.3", + "acorn": "^8.8.2", + "commander": "^2.20.0", + "source-map-support": "~0.5.20" + }, + "bin": { + "terser": "bin/terser" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "5.3.14", + "resolved": "https://registry.npmmirror.com/terser-webpack-plugin/-/terser-webpack-plugin-5.3.14.tgz", + "integrity": "sha512-vkZjpUjb6OMS7dhV+tILUW6BhpDR7P2L/aQSAv+Uwk+m8KATX9EccViHTJR2qDtACKPIYndLGCyl3FMo+r2LMw==", + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.25", + "jest-worker": "^27.4.5", + "schema-utils": "^4.3.0", + "serialize-javascript": "^6.0.2", + "terser": "^5.31.1" + }, + "engines": { + "node": ">= 10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.1.0" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "uglify-js": { + "optional": true + } + } + }, + "node_modules/thunky": { + "version": "1.1.0", + "resolved": "https://registry.npmmirror.com/thunky/-/thunky-1.1.0.tgz", + "integrity": "sha512-eHY7nBftgThBqOyHGVN+l8gF0BucP09fMo0oO/Lb0w1OF80dJv+lDVpXG60WMQvkcxAkNybKsrEIE3ZtKGmPrA==", + "dev": true, + "license": "MIT" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmmirror.com/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/ts-loader": { + "version": "9.5.2", + "resolved": "https://registry.npmmirror.com/ts-loader/-/ts-loader-9.5.2.tgz", + "integrity": "sha512-Qo4piXvOTWcMGIgRiuFa6nHNm+54HbYaZCKqc9eeZCLRy3XqafQgwX2F7mofrbJG3g7EEb+lkiR+z2Lic2s3Zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.1.0", + "enhanced-resolve": "^5.0.0", + "micromatch": "^4.0.0", + "semver": "^7.3.4", + "source-map": "^0.7.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "typescript": "*", + "webpack": "^5.0.0" + } + }, + "node_modules/ts-loader/node_modules/source-map": { + "version": "0.7.4", + "resolved": "https://registry.npmmirror.com/source-map/-/source-map-0.7.4.tgz", + "integrity": "sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">= 8" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmmirror.com/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD" + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmmirror.com/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/typescript": { + "version": "5.8.2", + "resolved": "https://registry.npmmirror.com/typescript/-/typescript-5.8.2.tgz", + "integrity": "sha512-aJn6wq13/afZp/jT9QZmwEjDqqvSGp1VT5GVg+f/t6/oVyrgXM6BY1h9BRh/O5p3PlUPAe+WuiEZOmb/49RqoQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmmirror.com/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", + "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/utila": { + "version": "0.4.0", + "resolved": "https://registry.npmmirror.com/utila/-/utila-0.4.0.tgz", + "integrity": "sha512-Z0DbgELS9/L/75wZbro8xAnT50pBVFQZ+hUEueGDU5FN51YSCYM+jdxsfCiHjwNP/4LCDD0i/graKpeBnOXKRA==", + "dev": true, + "license": "MIT" + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmmirror.com/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmmirror.com/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "dev": true, + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmmirror.com/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/watchpack": { + "version": "2.4.2", + "resolved": "https://registry.npmmirror.com/watchpack/-/watchpack-2.4.2.tgz", + "integrity": "sha512-TnbFSbcOCcDgjZ4piURLCbJ3nJhznVh9kw6F6iokjiFPl8ONxe9A6nMDVXDiNbrSfLILs6vB07F7wLBrwPYzJw==", + "license": "MIT", + "dependencies": { + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.1.2" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/wbuf": { + "version": "1.7.3", + "resolved": "https://registry.npmmirror.com/wbuf/-/wbuf-1.7.3.tgz", + "integrity": "sha512-O84QOnr0icsbFGLS0O3bI5FswxzRr8/gHwWkDlQFskhSPryQXvrTMxjxGP4+iWYoauLoBvfDpkrOauZ+0iZpDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimalistic-assert": "^1.0.0" + } + }, + "node_modules/webpack": { + "version": "5.98.0", + "resolved": "https://registry.npmmirror.com/webpack/-/webpack-5.98.0.tgz", + "integrity": "sha512-UFynvx+gM44Gv9qFgj0acCQK2VE1CtdfwFdimkapco3hlPCJ/zeq73n2yVKimVbtm+TnApIugGhLJnkU6gjYXA==", + "license": "MIT", + "dependencies": { + "@types/eslint-scope": "^3.7.7", + "@types/estree": "^1.0.6", + "@webassemblyjs/ast": "^1.14.1", + "@webassemblyjs/wasm-edit": "^1.14.1", + "@webassemblyjs/wasm-parser": "^1.14.1", + "acorn": "^8.14.0", + "browserslist": "^4.24.0", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^5.17.1", + "es-module-lexer": "^1.2.1", + "eslint-scope": "5.1.1", + "events": "^3.2.0", + "glob-to-regexp": "^0.4.1", + "graceful-fs": "^4.2.11", + "json-parse-even-better-errors": "^2.3.1", + "loader-runner": "^4.2.0", + "mime-types": "^2.1.27", + "neo-async": "^2.6.2", + "schema-utils": "^4.3.0", + "tapable": "^2.1.1", + "terser-webpack-plugin": "^5.3.11", + "watchpack": "^2.4.1", + "webpack-sources": "^3.2.3" + }, + "bin": { + "webpack": "bin/webpack.js" + }, + "engines": { + "node": ">=10.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependenciesMeta": { + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-cli": { + "version": "5.1.4", + "resolved": "https://registry.npmmirror.com/webpack-cli/-/webpack-cli-5.1.4.tgz", + "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@discoveryjs/json-ext": "^0.5.0", + "@webpack-cli/configtest": "^2.1.1", + "@webpack-cli/info": "^2.0.2", + "@webpack-cli/serve": "^2.0.5", + "colorette": "^2.0.14", + "commander": "^10.0.1", + "cross-spawn": "^7.0.3", + "envinfo": "^7.7.3", + "fastest-levenshtein": "^1.0.12", + "import-local": "^3.0.2", + "interpret": "^3.1.1", + "rechoir": "^0.8.0", + "webpack-merge": "^5.7.3" + }, + "bin": { + "webpack-cli": "bin/cli.js" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "5.x.x" + }, + "peerDependenciesMeta": { + "@webpack-cli/generators": { + "optional": true + }, + "webpack-bundle-analyzer": { + "optional": true + }, + "webpack-dev-server": { + "optional": true + } + } + }, + "node_modules/webpack-cli/node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmmirror.com/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14" + } + }, + "node_modules/webpack-dev-middleware": { + "version": "5.3.4", + "resolved": "https://registry.npmmirror.com/webpack-dev-middleware/-/webpack-dev-middleware-5.3.4.tgz", + "integrity": "sha512-BVdTqhhs+0IfoeAf7EoH5WE+exCmqGerHfDM0IL096Px60Tq2Mn9MAbnaGUe6HiMa41KMCYF19gyzZmBcq/o4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "colorette": "^2.0.10", + "memfs": "^3.4.3", + "mime-types": "^2.1.31", + "range-parser": "^1.2.1", + "schema-utils": "^4.0.0" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.0.0 || ^5.0.0" + } + }, + "node_modules/webpack-dev-server": { + "version": "4.15.1", + "resolved": "https://registry.npmmirror.com/webpack-dev-server/-/webpack-dev-server-4.15.1.tgz", + "integrity": "sha512-5hbAst3h3C3L8w6W4P96L5vaV0PxSmJhxZvWKYIdgxOQm8pNZ5dEOmmSLBVpP85ReeyRt6AS1QJNyo/oFFPeVA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/bonjour": "^3.5.9", + "@types/connect-history-api-fallback": "^1.3.5", + "@types/express": "^4.17.13", + "@types/serve-index": "^1.9.1", + "@types/serve-static": "^1.13.10", + "@types/sockjs": "^0.3.33", + "@types/ws": "^8.5.5", + "ansi-html-community": "^0.0.8", + "bonjour-service": "^1.0.11", + "chokidar": "^3.5.3", + "colorette": "^2.0.10", + "compression": "^1.7.4", + "connect-history-api-fallback": "^2.0.0", + "default-gateway": "^6.0.3", + "express": "^4.17.3", + "graceful-fs": "^4.2.6", + "html-entities": "^2.3.2", + "http-proxy-middleware": "^2.0.3", + "ipaddr.js": "^2.0.1", + "launch-editor": "^2.6.0", + "open": "^8.0.9", + "p-retry": "^4.5.0", + "rimraf": "^3.0.2", + "schema-utils": "^4.0.0", + "selfsigned": "^2.1.1", + "serve-index": "^1.9.1", + "sockjs": "^0.3.24", + "spdy": "^4.0.2", + "webpack-dev-middleware": "^5.3.1", + "ws": "^8.13.0" + }, + "bin": { + "webpack-dev-server": "bin/webpack-dev-server.js" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^4.37.0 || ^5.0.0" + }, + "peerDependenciesMeta": { + "webpack": { + "optional": true + }, + "webpack-cli": { + "optional": true + } + } + }, + "node_modules/webpack-dev-server/node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmmirror.com/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/webpack-merge": { + "version": "5.10.0", + "resolved": "https://registry.npmmirror.com/webpack-merge/-/webpack-merge-5.10.0.tgz", + "integrity": "sha512-+4zXKdx7UnO+1jaN4l2lHVD+mFvnlZQP/6ljaJVb4SZiwIKeUnrT5l0gkT8z+n4hKpC+jpOv6O9R+gLtag7pSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "clone-deep": "^4.0.1", + "flat": "^5.0.2", + "wildcard": "^2.0.0" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/webpack-sources": { + "version": "3.2.3", + "resolved": "https://registry.npmmirror.com/webpack-sources/-/webpack-sources-3.2.3.tgz", + "integrity": "sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==", + "license": "MIT", + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/websocket-driver": { + "version": "0.7.4", + "resolved": "https://registry.npmmirror.com/websocket-driver/-/websocket-driver-0.7.4.tgz", + "integrity": "sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "http-parser-js": ">=0.5.1", + "safe-buffer": ">=5.1.0", + "websocket-extensions": ">=0.1.1" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmmirror.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmmirror.com/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wildcard": { + "version": "2.0.1", + "resolved": "https://registry.npmmirror.com/wildcard/-/wildcard-2.0.1.tgz", + "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmmirror.com/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.13.0", + "resolved": "https://registry.npmmirror.com/ws/-/ws-8.13.0.tgz", + "integrity": "sha512-x9vcZYTrFPC7aSIbj7sRCYo7L/Xb8Iy+pW0ng0wt2vCJv7M9HOMy0UoN3rr+IFC7hb7vXoqS+P9ktyLLLhO+LA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} \ No newline at end of file diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/package.json b/plugins/tensorboard-plugins/tb_graph_ascend/fe/package.json new file mode 100644 index 0000000000000000000000000000000000000000..d5bf7321825ef2b085828176392addd974620162 --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/package.json @@ -0,0 +1,76 @@ +{ + "name": "tb-graph-ascend", + "version": "0.1.0", + "private": "true", + "main": "index.js", + "scripts": { + "dev": "webpack serve --config webpack.dev.js", + "buildLinux": "cross-env NODE_ENV=production webpack && cp dist/index.html ../server/static/", + "buildWin": "cross-env NODE_ENV=production webpack && copy dist\\index.html ..\\server\\static\\", + "prettier": "prettier --config ./.prettierrc --write ./src/**/*.ts" + }, + "devDependencies": { + "@types/d3": "5.7.2", + "@types/lodash": "^4.14.172", + "@types/node": "^16.4.13", + "@types/offscreencanvas": "^2019.6.3", + "@types/requirejs": "^2.1.33", + "@types/resize-observer-browser": "^0.1.6", + "@types/three": "^0.131.0", + "html-loader": "^5.1.0", + "html-webpack-plugin": "^5.6.3", + "inline-chunk-html-plugin": "^1.1.1", + "ts-loader": "^9.5.1", + "tslib": "^2.6.2", + "typescript": "^5.4.5", + "webpack": "^5.96.1", + "webpack-cli": "^5.1.4", + "webpack-dev-server": "4.15.1", + "ws": "8.13.0" + }, + "dependencies": { + "@polymer/decorators": "^3.0.0", + "@polymer/iron-behaviors": "^3.0.1", + "@polymer/iron-collapse": "^3.0.1", + "@polymer/iron-icon": "^3.0.1", + "@polymer/iron-icons": "^3.0.1", + "@polymer/iron-iconset-svg": "^3.0.1", + "@polymer/iron-list": "^3.1.0", + "@polymer/iron-resizable-behavior": "^3.0.1", + "@polymer/paper-behaviors": "^3.0.1", + "@polymer/paper-button": "^3.0.1", + "@polymer/paper-checkbox": "^3.1.0", + "@polymer/paper-dialog": "^3.0.1", + "@polymer/paper-dropdown-menu": "^3.1.0", + "@polymer/paper-icon-button": "^3.0.2", + "@polymer/paper-item": "^3.0.1", + "@polymer/paper-listbox": "^3.0.1", + "@polymer/paper-progress": "^3.0.1", + "@polymer/paper-tooltip": "^3.0.1", + "@polymer/polymer": "^3.5.1", + "@types/lodash": "^4.17.1", + "@vaadin/button": "24.6.5", + "@vaadin/combo-box": "24.6.5", + "@vaadin/context-menu": "24.6.5", + "@vaadin/details": "24.6.5", + "@vaadin/grid": "24.6.5", + "@vaadin/icon": "24.6.5", + "@vaadin/icons": "24.6.5", + "@vaadin/notification": "24.6.5", + "@vaadin/progress-bar": "24.6.5", + "@vaadin/select": "24.6.5", + "@vaadin/tabs": "24.6.5", + "@vaadin/tabsheet": "24.6.5", + "@vaadin/text-field": "24.6.5", + "@vaadin/tooltip": "24.6.5", + "axios": "^1.8.4", + "clean-webpack-plugin": "^4.0.0", + "cross-env": "^7.0.3", + "css-loader": "^7.1.2", + "d3": "5.7.0", + "dagre": "^0.8.5", + "lodash": "^4.17.21", + "prettier": "^3.4.2", + "style-loader": "^4.0.0" + } +} diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/constant.ts b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/constant.ts new file mode 100644 index 0000000000000000000000000000000000000000..145aa943805e35341a4453b44b7ae6495c27fb64 --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/constant.ts @@ -0,0 +1,80 @@ +/* Copyright (c) 2025, Huawei Technologies. + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +// NPU侧模型的节点前缀 +export const NPU_PREFIX = 'N___'; +// 标杆侧模型的节点前缀 +export const BENCH_PREFIX = 'B___'; +// 未匹配节点颜色 +export const UNMATCHED_COLOR = '#C7C7C7'; + +// 双图下单个图形的最小宽度 +export const MIN_GRAPG_WIDTH = 200; + +// 预设颜色 +export const defaultColorSetting = [ + { key: '#FFFCF3', values: [0, 0.2] }, + { key: '#FFEDBE', values: [0.2, 0.4] }, + { key: '#FFDC7F', values: [0.4, 0.6] }, + { key: '#FFC62E', values: [0.6, 0.8] }, + { key: '#ff704d', values: [0.8, 1] }, +]; +// 预设颜色设置项 +export const defaultColorSelects = [{ key: 'NaN', values: [NaN, NaN] }]; + +export enum NODE_TYPE { + MODULE = 0, // 圆角矩形，有可展开，不可展开两种情况，可展开的宽度较宽，不可展开，宽度较窄 + UNEXPAND_NODE = 1, // 椭圆形，不可展开,API + API_LIST = 9, // API列表 + MULTI_COLLECTION = 8, // 融合算子 +} + +// 渲染信息 +export const DURATION_TIME = 160; // 动画时间 +export const SELECTED_STROKE_COLOR = 'rgb(31, 63, 207)'; // 选中节点颜色 +export const BENCH_NODE_COLOR = 'rgb(236, 235, 235)'; // 基准模型节点颜色 +export const BENCH_STROKE_COLOR = 'rgb(161, 161, 161)'; // 基准模型边框颜色 +export const NO_MATCHED_NODE_COLOR = 'rgb(199, 199, 199)'; // 未匹配节点颜色 +export const BASE_NODE_COLOR = 'rgb(255, 255, 255)'; // 基准节点颜色，没有精度信息、API、FUSION的填充色 +export const STROKE_WIDTH = 1.5; // 边框宽度 +export const SELECTED_STROKE_WIDTH = 2; // 边框颜色 + +export const MOVE_STEP = 40; // 移动步长 +export const SCALE_STEP = 0.2; // 缩放步长 + +export const MAX_SCALE = 3; // 最大缩放 +export const MIN_SCALE = 1; // 最小缩放 + +// 溢出检测颜色 +export enum OVERFLOW_COLOR { + medium = ' #B6C7FC', + high = ' #7E96F0', + critical = ' #4668B8', + default = 'rgb(199, 199, 199)', +} + +export const NODE_TYPE_STYLES = { + // 节点样式 + [NODE_TYPE.MODULE]: { strokeDasharray: '20,0', rx: '5', ry: '5' }, + [NODE_TYPE.UNEXPAND_NODE]: { strokeDasharray: '20,0', rx: '50%', ry: '50%', fontSize: 6 }, + [NODE_TYPE.API_LIST]: { strokeDasharray: '15,1', rx: '5', ry: '5' }, + [NODE_TYPE.MULTI_COLLECTION]: { strokeDasharray: '2,1', rx: '5', ry: '5' }, +}; + +export const PREFIX_MAP = { + Single: '', + NPU: NPU_PREFIX, + Bench: BENCH_PREFIX, +}; diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/graph-board-layout/index.ts b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/graph-board-layout/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..000bb979e86acd56d486d4b1512b73ac137ad25e --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/graph-board-layout/index.ts @@ -0,0 +1,168 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +import { customElement } from '@polymer/decorators'; +import { html, PolymerElement } from '@polymer/polymer'; +import { DarkModeMixin } from '../../polymer/dark_mode_mixin'; +import './tensorboardColor'; + +@customElement('graph-board-layout') +class TfDashboardLayout extends DarkModeMixin(PolymerElement) { + static readonly template = html` +
+ +
+ +
+
+ +
+ +
+ + + `; + + _toggleSidebar(): void { + // 通过 ID 获取元素并隐藏 + const sidebar = this.shadowRoot?.querySelector('#sidebar'); + const sidebarToggle = this.shadowRoot?.querySelector('#sidebar-toggle'); + // 检查并切换 display 样式 + if (sidebar) { + sidebar?.classList.toggle('sider-hidden'); // 改为显示 + sidebarToggle?.classList.toggle('sidebar-toggle-fold'); // 改变箭头方向 + } + } +} diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/graph-board-layout/tensorboardColor.ts b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/graph-board-layout/tensorboardColor.ts new file mode 100644 index 0000000000000000000000000000000000000000..e76ed139f9d2956cad1cb1c782358b9ee1975c00 --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/common/graph-board-layout/tensorboardColor.ts @@ -0,0 +1,57 @@ +/* Copyright 2016 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +const style = document.createElement('style'); +style.setAttribute('is', 'custom-style'); +style.textContent = ` + :root { + --tb-orange-weak: #ffa726; + --tb-orange-strong: #f57c00; + --tb-orange-dark: #dc7320; + --tb-grey-darker: #e2e2e2; + --tb-grey-lighter: #f3f3f3; + --tb-ui-dark-accent: #757575; + --tb-ui-light-accent: #e0e0e0; + --tb-ui-border: var(--paper-grey-300); + --tb-graph-faded: #e0d4b3; + --tb-secondary-text-color: var(--paper-grey-800); + --tb-raised-button-shadow-color: rgba(0, 0, 0, 0.2); + --primary-background-color: #fff; + --secondary-background-color:rgb(247, 247, 247); + --tb-layout-background-color: #f5f5f5; + --tb-link: #1976d2; /* material blue 700. */ + --tb-link-visited: #7b1fa2; /* material purple 700. */ + } + + :root .dark-mode { + --tb-ui-border: var(--paper-grey-700); + --tb-ui-dark-accent: var(--paper-grey-400); + --tb-ui-light-accent: var(--paper-grey-600); + --tb-secondary-text-color: var(--paper-grey-400); + --tb-raised-button-shadow-color: rgba(255, 255, 255, 0.5); + --primary-text-color: #fff; + --secondary-text-color: var(--paper-grey-400); + --primary-background-color: #303030; /* material grey A400. */ + --secondary-background-color: #3a3a3a; + --tb-layout-background-color: #3a3a3a; + --tb-link: #42a5f5; /* material blue 400. */ + --tb-link-visited: #ba68c8; /* material purple 300. */ + /* Overrides paper-material */ + --shadow-elevation-2dp_-_box-shadow: 0 2px 2px 0 rgba(255, 255, 255, 0.14), + 0 1px 5px 0 rgba(255, 255, 255, 0.12), + 0 3px 1px -2px rgba(255, 255, 255, 0.2); + } +`; +document.head.appendChild(style); diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/index.ts b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..bb95f043d9b9cf9baf39e1ffdbd5cf668454e2d8 --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/index.ts @@ -0,0 +1,346 @@ +/* Copyright (c) 2025, Huawei Technologies. + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { customElement, observe, property } from '@polymer/decorators'; +import { html, PolymerElement } from '@polymer/polymer'; +import { LegacyElementMixin } from '../polymer/legacy_element_mixin'; +import useGraphAscend from './useGraphAscend'; +import { formatBytes, safeJSONParse } from '../utils'; +import '../graph_board/index'; +import '../graph_info_board/index'; +import '../graph_controls_board/index'; +import '../common/graph-board-layout'; +import type { SelectionType, ProgressType, GraphConfigType, GraphAllNodeType, NodeListType, UnmatchedNodeType } from './type'; + +@customElement('graph-ascend') +class TfGraphDashboard extends LegacyElementMixin(PolymerElement) { + static readonly template = html` + + + +
+
+ +
+ + +
+ + + + `; + + @property({ type: Array }) + metaDir: Record = {}; + + @property({ type: Object, notify: true }) + selection: SelectionType | null = null; + + @property({ type: Object, notify: true }) + nodelist: any; + + @property({ type: Object, notify: true }) + unmatched: any; + + @property({ type: Object, notify: true }) + matchedlist: any; + + @property({ type: String, notify: true }) + selectedNode: string = ''; + + @property({ type: String, notify: true }) + jumpToNode: string = ''; + + @property({ type: Object, notify: true }) + colors: any; + + @property({ type: Boolean, notify: true }) + isOverflowFilter: boolean = false; + + @property({ type: Object }) + progressData: ProgressType = { progress: 0, progressValue: 0, done: false }; + + @property({ type: Boolean }) + isSingleGraph: boolean = false; + + @property({ type: Object }) + microsteps: any; + + @property({ type: Array }) + overflowcheck; + + @property({ type: Object }) + tooltips: object = {}; + + @property({ type: Object }) + colorset: object = {}; + + @property({ type: Object }) + npuMatchNodes: object = {}; + + @property({ type: Object }) + benchMatchNodes: object = {}; + + @property({ type: Object }) + matchedConfigFiles: string[] = []; + + private currentSelection: SelectionType | null = null; + private useGraphAscend = useGraphAscend(); + private eventSource: EventSource | null = null; + + @observe('selection') + updateGraphData = () => { + if (!this.selection?.run || !this.selection?.tag) { + return; + } + if (this.currentSelection?.run !== this.selection?.run || this.currentSelection?.tag !== this.selection?.tag) { + this.loadGraphData(this.selection.run, this.selection.tag); + } else if (this.currentSelection?.microStep !== this.selection?.microStep) { + this.initGraphBoard(); // 只改变microsteps时，不重新加载图数据 + this.loadGraphAllNodeList(this.selection.run, this.selection.tag, this.selection.microStep); + } + this.currentSelection = this.selection; + }; + + override async ready(): Promise { + super.ready(); + const metaDir = await this.useGraphAscend.loadGraphFileInfoList(); + this.set('metaDir', metaDir); + document.addEventListener( + 'contextMenuTag-changed', + (event: any) => this.set('jumpToNode', event.detail?.nodeName), + { passive: true }, + ); + } + + loadGraphData = (run, tag) => { + if (this.eventSource) { + this.eventSource.close(); + this.eventSource = null; + } + this.eventSource = new EventSource(`loadGraphData?run=${run}&tag=${tag}`); + this.eventSource.onmessage = async (e) => { + const data = safeJSONParse(e.data); + if (data.error) { + this.progreesError('初始化图失败', data.error); + } + if (data.status === 'reading') { + this.progressReading('正在读取文件', data); + } + if (data.status === 'loading') { + if (data.done) { + this.eventSource?.close(); + this.eventSource = null; + try { + await Promise.all([ + this.loadGraphConfig(this.selection?.run, this.selection?.tag), + this.loadGraphAllNodeList( + this.selection?.run, + this.selection?.tag, + this.selection?.microStep, + ), + ]); + this.initGraphBoard(); // 先读取配置，再加载图,顺序很重要 + this.progreesLoading('初始化完成', '请稍后', data); + } catch (error) { + this.progreesError('初始化图失败', error); + } + } else { + this.progreesLoading('正在解析文件', '正在初始化模型，请稍后.', data); + } + } + }; + + this.eventSource.onerror = (e) => { + if (!this.progressData || !this.progressData.done) { + this.progreesError('连接中断', '请检查网络连接'); + } + this.eventSource?.close(); + }; + }; + + loadGraphConfig = async (run, tag) => { + const { success, data } = await this.useGraphAscend.loadGraphConfig(run, tag); + const config = data as GraphConfigType; + if (success) { + this.set('colors', config.colors); + this.set('tooltips', config.tooltips); + this.set('overflowcheck', config.overflowCheck); + this.set('colorset', Object.entries(config.colors || {})); + this.set('isSingleGraph', config.isSingleGraph); + this.set('matchedConfigFiles', ['未选择', ...config.matchedConfigFiles]); + const microstepsCount = Number(config.microSteps); + if (microstepsCount) { + const microstepsArray = Array.from({ length: microstepsCount + 1 }, (_, index) => ({ + label: index === 0 ? 'ALL' : String(index - 1), + value: index - 1, + })); + this.set('microsteps', microstepsArray); + } else { + this.set('microsteps', []); + } + } + }; + + loadGraphAllNodeList = async (run, tag, microStep) => { + const { success, data } = await this.useGraphAscend.loadGraphAllNodeList(run, tag, microStep); + const allNodeList = data as GraphAllNodeType; + if (success) { + const nodelist = {} as NodeListType; + const unmatched = {} as UnmatchedNodeType; + if (this.isSingleGraph) { + nodelist.npu = allNodeList?.npuNodeList; + } else { + nodelist.npu = allNodeList?.npuNodeList; + nodelist.bench = allNodeList?.benchNodeList; + unmatched.npuNodeList = allNodeList?.npuUnMatchNodes; + unmatched.benchNodeList = allNodeList?.benchUnMatchNodes; + } + this.set('npuMatchNodes', allNodeList?.npuMatchNodes); + this.set('benchMatchNodes', allNodeList?.benchMatchNodes); + this.set('nodelist', nodelist); + this.set('unmatched', unmatched); + } + }; + + initGraphBoard = () => { + (this.shadowRoot?.querySelector('#graph-board') as any)?.initGraphHierarchy(this.jumpToNode); + if (this.jumpToNode) { + this.set('selectedNode', this.jumpToNode); + this.set('jumpToNode', ''); + } + }; + + onFitTap(): void { + (this.shadowRoot?.querySelector('#graph-board') as any).fitScreen(); + } + + progressReading = (title, data) => { + data.progressValue = data.done ? 1 : data.progress / 100.0; + data.size = formatBytes(data.size); + data.read = formatBytes(data.read); + data.title = title; + data.info = `文件大小: ${data.size}, 已读取: ${data.read}`; + this.set('progressData', data); + }; + + progreesLoading = (title, info, progressData) => { + const data = { + ...progressData, + title, + info, + }; + data.progressValue = progressData.done ? 1 : progressData.progress / 100.0; + this.set('progressData', data); + }; + + progreesError = (title, info) => { + const data = { + ...this.progressData, + title, + info, + }; + this.updateStyles({ + '--progress-background-color': 'red', + '--progress-color': 'red', + }); + this.set('progressData', data); + }; +} diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/type/index.d.ts b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/type/index.d.ts new file mode 100644 index 0000000000000000000000000000000000000000..7b080e2efc1cbd8c03174129501d170203f1927d --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/type/index.d.ts @@ -0,0 +1,60 @@ +/* Copyright (c) 2025, Huawei Technologies. + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +export interface ProgressType { + progress?: number; + progressValue?: number; + size?: number; + read?: number; + done?: boolean; +} + +export interface SelectionType { + run: string; + tag: string; + microStep: number; + +} + +export interface GraphConfigType { + tooltips: string; + colors: Record; + overflowCheck: boolean; + microSteps: number; + isSingleGraph: boolean; + matchedConfigFiles: string[]; +} + +export interface GraphAllNodeType { + npuNodeList: string[]; + benchNodeList: string[]; + npuUnMatchNodes: string[]; + benchUnMatchNodes: string[]; + npuMatchNodes: string[]; + benchMatchNodes: string[]; +} + +export interface NodeListType { + npu: string[]; + bench: string[]; +} + +export interface UnmatchedNodeType { + npuNodeList: string[]; + benchNodeList: string[]; +} \ No newline at end of file diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/useGraphAscend.ts b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/useGraphAscend.ts new file mode 100644 index 0000000000000000000000000000000000000000..d172093c6c7e068d04465d44e595b672012b6e4b --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_ascend/useGraphAscend.ts @@ -0,0 +1,56 @@ +/* Copyright (c) 2025, Huawei Technologies. + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import request from '../utils/request'; +const useGraphAscend = () => { + const loadGraphFileInfoList = async (): Promise => { + try { + const result = await request({ url: 'load_meta_dir', method: 'GET' }); + return result; + } catch (err) { + return { + success: false, + error: '加载文件信息失败', + }; + } + }; + const loadGraphConfig = async (runName: string, tagName: string): Promise => { + const params = { + run: runName, + tag: tagName, + }; + + const result = await request({ url: 'loadGraphConfigInfo', method: 'GET', params: params }); // 获取异步的 ArrayBuffer + return result; + }; + + const loadGraphAllNodeList = async (runName: string, tagName: string, microStep: number): Promise => { + const params = { + run: runName, + tag: tagName, + microStep: microStep, + }; + + const result = await request({ url: 'loadGraphAllNodeList', method: 'GET', params: params }); // 获取异步的 ArrayBuffer + return result; + }; + + return { + loadGraphConfig, + loadGraphAllNodeList, + loadGraphFileInfoList, + }; +}; +export default useGraphAscend; diff --git a/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_board/components/hierarchy/index.ts b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_board/components/hierarchy/index.ts new file mode 100644 index 0000000000000000000000000000000000000000..21e8a5df69a402f7cbee41240de42394972f98bb --- /dev/null +++ b/plugins/tensorboard-plugins/tb_graph_ascend/fe/src/graph_board/components/hierarchy/index.ts @@ -0,0 +1,711 @@ +/* Copyright (c) 2025, Huawei Technologies. + * All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { PolymerElement, html } from '@polymer/polymer'; +import { customElement, property, observe } from '@polymer/decorators'; +import * as d3 from 'd3'; +import useGraph from './useGraph'; +import { changeGraphPosition } from '../../../utils/index'; +import { parseTransform } from '../../../utils/index'; +import { isEmpty, throttle } from 'lodash'; +import * as minimap from '../minimap/minimap'; +import { + NPU_PREFIX, + BENCH_PREFIX, + MOVE_STEP, + SCALE_STEP, + NODE_TYPE, + MAX_SCALE, + MIN_SCALE, + PREFIX_MAP, +} from '../../../common/constant'; +import '../minimap/index'; +import '@vaadin/context-menu'; +import { Notification } from '@vaadin/notification'; +import type { UseGraphType } from '../../type'; +import type { HierarchyNodeType, ContextMenuItem, PreProcessDataConfigType, GraphType } from '../../type'; +import type { ContextMenuItemSelectedEvent } from '@vaadin/context-menu'; + +const EXPAND_MATCHED_NODE = 1; +const DATA_COMMUNICATION = 2; +const DATA_COMMUNICATION_TYEPE = { + send: '数据发送', + receive: '数据接收', + send_receive: '数据发送接收', +}; +@customElement('graph-hierarchy') +class Hierarchy extends PolymerElement { + static readonly template = html` + +
+ + + + + + +