diff --git a/plugins/mindstudio-insight-plugins/CMakeLists.txt b/plugins/mindstudio-insight-plugins/CMakeLists.txt index 45e7091851fe12a175cdc44fafb5cb03160f64e5..6ecab5aca3a3a7b61490747912134d9024740ea0 100644 --- a/plugins/mindstudio-insight-plugins/CMakeLists.txt +++ b/plugins/mindstudio-insight-plugins/CMakeLists.txt @@ -3,5 +3,11 @@ project(MindStudio_Board) set(PROJECT_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin_core/cmake/mind_expression.cmake) add_subdirectory(proto) + +if (${CMAKE_BUILD_TYPE} MATCHES "Debug") + message(STATUS "Open tools generate") + add_compile_definitions(PLUGINS_DIR="${PROJECT_ROOT_DIR}/output/plugins") + add_subdirectory(tools/httpServer) +endif () +add_subdirectory(plugin_core) add_subdirectory(Scalar) -add_subdirectory(plugin_core) \ No newline at end of file diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/CMakeLists.txt b/plugins/mindstudio-insight-plugins/Scalar/server/CMakeLists.txt index 37f59b38312a262c6b355f3b5eff9ad30846e2ce..a54e3e386608d2db1aa2d01be50d21b05ef9dcfc 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/CMakeLists.txt +++ b/plugins/mindstudio-insight-plugins/Scalar/server/CMakeLists.txt @@ -13,11 +13,11 @@ set(CMAKE_CXX_STANDARD 17) set(CMAKE_C_STANDARD 11) set(HOME_DIR ${PROJECT_SOURCE_DIR}) -set(EXECUTABLE_OUTPUT_PATH ${SCALAR_PROJECT_ROOT_DIR}/output/bin) -set(LIBRARY_OUTPUT_PATH ${SCALAR_PROJECT_ROOT_DIR}/output/bin) +set(EXECUTABLE_OUTPUT_PATH ${PROJECT_ROOT_DIR}/output/plugins) +set(LIBRARY_OUTPUT_PATH ${PROJECT_ROOT_DIR}/output/plugins) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2 -ftrapv -fstack-protector-strong -fPIE -fPIC") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector-all -D_FORTIFY_SOURCE=2 -O2 -ftrapv -fstack-protector-strong -fPIE -fPIC") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstack-protector-all -D_FORTIFY_SOURCE=2 -O0 -g -ftrapv -fstack-protector-strong -fPIE -fPIC") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fstack-protector-all -D_FORTIFY_SOURCE=2 -O0 -g -ftrapv -fstack-protector-strong -fPIE -fPIC") if (${CMAKE_BUILD_TYPE} MATCHES "Debug") message(STATUS "Enable debug symbol table, change optimization level to 0") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/CMakeLists.txt b/plugins/mindstudio-insight-plugins/Scalar/server/src/CMakeLists.txt index 1b402ba42896b813601964274b79e1f42beb801f..8abb0d35a97616fa193d3afc5f5388ebce3c4204 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/CMakeLists.txt +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/CMakeLists.txt @@ -2,41 +2,47 @@ set(SRC_HOME_DIR ${HOME_DIR}/src) aux_source_directory(plugin SCALAR_SRC_LIST) aux_source_directory(plugin/Handler SCALAR_SRC_LIST) aux_source_directory(parser SCALAR_SRC_LIST) +aux_source_directory(Sampler SCALAR_SRC_LIST) aux_source_directory(${PROJECT_ROOT_DIR}/proto SCALAR_SRC_LIST) aux_source_directory(defs SCALAR_SRC_LIST) aux_source_directory(FileManager SCALAR_SRC_LIST) aux_source_directory(GraphManager SCALAR_SRC_LIST) aux_source_directory(Util SCALAR_SRC_LIST) -aux_source_directory(${PROJECT_ROOT_DIR}/plugin_core/src SCALAR_SRC_LIST) +set(LOG_SRC ${PROJECT_ROOT_DIR}/plugin_core/src/Logger.cpp) list(APPEND ${PROJECT_NAME}_SRC ${PROTO_SRC} - ${SCALAR_SRC_LIST}) + ${SCALAR_SRC_LIST} + ${LOG_SRC}) include_directories(${SRC_HOME_DIR} - ${SRC_HOME_DIR}/plugin - ${SRC_HOME_DIR}/parser - ${PROJECT_ROOT_DIR}/proto + ${SRC_HOME_DIR}/plugin + ${SRC_HOME_DIR}/parser + ${SRC_HOME_DIR}/Sampler + ${PROJECT_ROOT_DIR}/proto + ${PROJECT_ROOT_DIR} ) set(LIBRARY_OUTPUT_PATH ${LIBRARY_OUTPUT_PATH}/${PROJECT_NAME}) -add_library(${PROJECT_NAME} PUBLIC ${${PROJECT_NAME}_SRC}) +add_library(${PROJECT_NAME} SHARED ${${PROJECT_NAME}_SRC}) target_include_directories(${PROJECT_NAME} PRIVATE ${${PROJECT_NAME}_H}) target_include_directories(${PROJECT_NAME} PRIVATE ${rapidjson_INC}) -if (${CMAKE_BUILD_TYPE} MATCHES "Debug") - target_compile_options(${PROJECT_NAME} PRIVATE -O0 -g) -endif () target_include_directories(${PROJECT_NAME} PRIVATE ${PROJECT_ROOT_DIR}/plugin_core/include) -target_link_libraries(${PROJECT_NAME} PRIVATE ${PROJECT_ROOT_DIR}/plugin_core/lib/libmsinsight.so) target_link_libraries(${PROJECT_NAME} PRIVATE mindboard::protobuf) +find_library(MSINSIGHT_LIB + NAMES msinsight + PATHS ${PROJECT_ROOT_DIR}/output/lib + NO_DEFAULT_PATH) +target_link_libraries(${PROJECT_NAME} PUBLIC ${MSINSIGHT_LIB}) set(CMAKE_CXX_VISIBILITY_PRESET default) if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") target_link_libraries(${PROJECT_NAME} PRIVATE stdc++fs) endif () -#-------- test -------- +#-------- ut test -------- if (ENABLE_TESTCASES OR ENABLE_CPP_ST) enable_testing() aux_source_directory(test TEST_SRC) + aux_source_directory(${PROJECT_ROOT_DIR}/plugin_core/src TEST_SRC) list(APPEND ${PROJECT_NAME}_TEST_SRC ${${PROJECT_NAME}_SRC} ${TEST_SRC}) @@ -45,16 +51,16 @@ if (ENABLE_TESTCASES OR ENABLE_CPP_ST) add_executable(${PROJECT_NAME}_test ${${PROJECT_NAME}_TEST_SRC}) target_include_directories(${PROJECT_NAME}_test PUBLIC ${${PROJECT_NAME}_TEST_H}) target_include_directories(${PROJECT_NAME}_test PUBLIC ${PROJECT_ROOT_DIR}/plugin_core/include) - if (CMAKE_SYSTEM_NAME STREQUAL "Linux") - target_link_libraries(${PROJECT_NAME}_test PRIVATE stdc++fs) - endif () + target_include_directories(${PROJECT_NAME}_test PUBLIC ${PROJECT_ROOT_DIR}) + target_include_directories(${PROJECT_NAME}_test PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/plugin) + target_include_directories(${PROJECT_NAME}_test PRIVATE ${PROJECT_ROOT_DIR}/rapidjson/include/rapidjson) + + target_link_libraries(${PROJECT_NAME}_test PRIVATE stdc++fs) target_link_libraries(${PROJECT_NAME}_test PRIVATE mindboard::protobuf) target_link_libraries(${PROJECT_NAME}_test PUBLIC gtest_main gmock_main) - target_link_libraries(${PROJECT_NAME}_test PUBLIC ${PROJECT_ROOT_DIR}/plugin_core/lib/libmsinsight.so) target_link_libraries(${PROJECT_NAME}_test PRIVATE mindboard::protobuf) - if (CMAKE_BUILD_TYPE STREQUAL "Debug") - target_compile_options(${PROJECT_NAME}_test PRIVATE -O0 -g) - endif () + target_link_libraries(${PROJECT_NAME}_test PUBLIC ${MSINSIGHT_LIB}) + target_compile_options(${PROJECT_NAME}_test PRIVATE -O0 -g) set_target_properties(${PROJECT_NAME}_test PROPERTIES EXCLUDE_FROM_ALL true) endif () \ No newline at end of file diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/FileManager/FileInfoManager.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/FileManager/FileInfoManager.cpp index 4f2c0b6d9cab2bc738c86d5fc5b92d5e6c0f96e5..06d64645870b922270479ca268776adf6c54a67f 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/FileManager/FileInfoManager.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/FileManager/FileInfoManager.cpp @@ -7,11 +7,12 @@ using namespace Insight::Scalar::FileInfo; using namespace Insight; -std::shared_ptr FileInfoManager::AddFile(const std::string &filePath, ParseDataType dataType) { +std::shared_ptr FileInfoManager::AddFile(const std::string &filePath, ParseDataType dataType) +{ if (filePaths_.count(filePath) != 0) { return fileInfoMap_[filePath]; } - auto fileInfo = std::make_shared(filePath, dataType); + auto fileInfo = std::make_shared(filePath, dataType); filePaths_.insert(filePath); fileInfoMap_[filePath] = fileInfo; return fileInfo; @@ -26,7 +27,8 @@ bool FileInfoManager::DelFileFromMap(std::string_view filePath) { return true; } -std::shared_ptr FileInfoManager::GetFileInfo(std::string_view filePath) { +std::shared_ptr FileInfoManager::GetFileInfo(std::string_view filePath) +{ if (filePaths_.count(filePath.data()) == 0) { return nullptr; } diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/FileManager/FileInfoManager.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/FileManager/FileInfoManager.h index 952afa0d3ae2618ad9f852dcb921030988d73232..42c6e43302df929f5e9386df758daadd9715be6b 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/FileManager/FileInfoManager.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/FileManager/FileInfoManager.h @@ -17,9 +17,9 @@ enum class UpdateType { DEL }; -struct FileInfo { - FileInfo(std::string filePath, ParseDataType dataType) : filePath_(std::move(filePath)), - parseDataType_(dataType) {}; +struct File { + File(std::string filePath, ParseDataType dataType) : filePath_(std::move(filePath)), parseDataType_(dataType) + {}; std::string filePath_; ParseDataType parseDataType_; uint64_t offSet_{0}; @@ -35,7 +35,7 @@ public: * @param dataType * @return */ - std::shared_ptr AddFile(const std::string &filePath, ParseDataType dataType); + std::shared_ptr AddFile(const std::string &filePath, ParseDataType dataType); /** * @brief delete file object, if success return true @@ -44,7 +44,7 @@ public: */ bool DelFileFromMap(std::string_view filePath); - std::shared_ptr GetFileInfo(std::string_view filePath); + std::shared_ptr GetFileInfo(std::string_view filePath); void Reset(); @@ -79,13 +79,14 @@ public: private: std::set filePaths_; - std::unordered_map> fileInfoMap_; + std::unordered_map> fileInfoMap_; std::unordered_map> createFileGroupByDir_; inline static std::map fileTypeMap_ = { {R"(out.tfevent)", Insight::Scalar::ParseDataType::TF_EVENT}, {R"(out.events.summary)", Insight::Scalar::ParseDataType::MINDSPORE_SUMMARY}, - {R"(worker_[0-9]+\.log)", Insight::Scalar::ParseDataType::TEXT_LOG} + {R"(worker_[0-9]+.*\.log)", Insight::Scalar::ParseDataType::TEXT_LOG}, + {R"(npu_.*\.log)", Insight::Scalar::ParseDataType::TEXT_LOG} }; }; } diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/GraphManager/GraphManager.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/GraphManager/GraphManager.cpp index 9f36990c7a7912a149960b5112fdf2227a38310b..9a6a61139b090d34b8a0ac4bdc4af383a7f43dc5 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/GraphManager/GraphManager.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/GraphManager/GraphManager.cpp @@ -12,6 +12,7 @@ using namespace Insight::Scalar::GraphOp; using namespace Insight; using namespace Protocol; +constexpr uint64_t MAX_POINT_PER_REQUEST = 30000; // avoid frontend performance problem, not pull data in one request std::optional Graph::GetFileData(const std::string &file, uint64_t offset, uint64_t sampleOffset, std::unique_ptr sampleParam) { @@ -23,7 +24,13 @@ Graph::GetFileData(const std::string &file, uint64_t offset, uint64_t sampleOffs } auto &dataSrc = dataMap_[file]; offset = offset > dataSrc.size() ? dataSrc.size() : offset; - std::copy(dataSrc.begin() + static_cast(offset), dataSrc.end(), std::back_inserter(data.graphData_)); + auto end = dataSrc.begin(); + if (offset + MAX_POINT_PER_REQUEST > dataSrc.size()) { + end = dataSrc.end(); + } else { + end = end + offset + MAX_POINT_PER_REQUEST - 1; + } + std::copy(dataSrc.begin() + static_cast(offset), end, std::back_inserter(data.graphData_)); if (!sampleParam->algorithm_.empty()) { UpdateSample(file, std::move(sampleParam)); GetSampledData(file, sampleOffset, data); diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/IRLowPassSampler.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/IRLowPassSampler.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fea044566d3ff6406d45455cc5c4ea40848bfb48 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/IRLowPassSampler.cpp @@ -0,0 +1,77 @@ +/* +* Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. +*/ +#include "IRLowPassSampler.h" +#include +#include +#include + +using namespace Insight::Scalar::Sample; + +void IRLowPassSampler::SetSampleParam(std::unique_ptr param) +{ + if (param == nullptr) { + return; + } + if (param->weight != weight_) { + last_ = 0.0; + numAccum_ = 0; + weight_ = param->weight; + } +} + +std::vector +IRLowPassSampler::Sample(const std::vector &original, uint32_t start, int32_t length) +{ + if (start > original.size() || weight_ == 0.0) { + return {}; + } + if (start == 0) // frontend reopen or fresh, need clean cache + { + last_ = 0.0; + numAccum_ = 0; + } + size_t dataLen = original.size() - start; + if (length != -1) // -1 + { + dataLen = length > dataLen ? dataLen : length; + } + std::vector result; + result.reserve(dataLen); + float firstValue = original[0].value_; + bool isConstant = std::all_of(original.begin(), original.end(), [&firstValue](const auto &point) { + return point.value_ == firstValue; + }); + for (size_t index = 0; index < dataLen; index++) { + auto &point = original[index]; + ScalarPoint sampledPoint{}; + sampledPoint.step_ = point.step_; + if (isConstant || std::isinf(point.value_) || std::isnan(point.value_)) { + sampledPoint.value_ = point.value_; + } else { + last_ = last_ * weight_ + (1 - weight_) * point.value_; + numAccum_++; + float debiasWeight = 1.0; + if (weight_ != 1.0) { + debiasWeight = debiasWeight - static_cast(pow(weight_, numAccum_)); + } + sampledPoint.value_ = last_ / debiasWeight; + } + result.emplace_back(sampledPoint); + } + return result; +} + +bool IRLowPassSampler::IsSameParam(SampleParam *param) +{ + if (param == nullptr) { + return false; + } + if (param->algorithm_ != algorithm_) { + return false; + } + if (param->weight != weight_) { + return false; + } + return true; +} diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/IRLowPassSampler.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/IRLowPassSampler.h new file mode 100644 index 0000000000000000000000000000000000000000..4b99132ab533dd1ae6051bef3c3c9d78943ad598 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/IRLowPassSampler.h @@ -0,0 +1,31 @@ +/* +* Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. +*/ +#ifndef MINDSTUDIO_BOARD_PLUGINS_MINDSTUDIO_INSIGHT_PLUGINS_SCALAR_SERVER_SRC_SAMPLER_IRLOWPASSSAMPLER_H_ +#define MINDSTUDIO_BOARD_PLUGINS_MINDSTUDIO_INSIGHT_PLUGINS_SCALAR_SERVER_SRC_SAMPLER_IRLOWPASSSAMPLER_H_ + +#include "SamplerBase.h" +#include + +using namespace Insight::Scalar::Sample; + +class IRLowPassSampler : public SamplerBase { +public: + IRLowPassSampler() + { + algorithm_ = "smoothing"; + }; + + void SetSampleParam(std::unique_ptr param) override; + + std::vector Sample(const std::vector &original, uint32_t start, int32_t length) override; + + bool IsSameParam(SampleParam *param) override; + +private: + float weight_{0.0}; + float last_{0.0}; + uint32_t numAccum_{0}; +}; + +#endif //MINDSTUDIO_BOARD_PLUGINS_MINDSTUDIO_INSIGHT_PLUGINS_SCALAR_SERVER_SRC_SAMPLER_IRLOWPASSSAMPLER_H_ diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/SamplerBase.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/SamplerBase.h new file mode 100644 index 0000000000000000000000000000000000000000..17612588e6b5b7c5353d2419be53484a254127c1 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/SamplerBase.h @@ -0,0 +1,45 @@ +/* +* Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. +*/ +#ifndef MINDSTUDIO_BOARD_PLUGINS_MINDSTUDIO_INSIGHT_PLUGINS_SCALAR_SERVER_SRC_SAMPLER_SAMPLERBASE_H_ +#define MINDSTUDIO_BOARD_PLUGINS_MINDSTUDIO_INSIGHT_PLUGINS_SCALAR_SERVER_SRC_SAMPLER_SAMPLERBASE_H_ + +#include +#include +#include "defs/ConceptDefs.h" + +using namespace Insight::Scalar; +namespace Insight::Scalar::Sample { +struct SampleParam { + SampleParam() = default; + + SampleParam(std::string algorithm, float weight) : algorithm_(std::move(algorithm)), weight(weight) + {} + + std::string algorithm_; + float weight{0}; +}; + +/** + * @brief use for data sampler + */ +class SamplerBase { +public: + SamplerBase() = default; + + virtual std::string GetAlgorithm() + { return algorithm_; } + + virtual ~SamplerBase() = default; + + virtual void SetSampleParam(std::unique_ptr param) = 0; + + virtual std::vector Sample(const std::vector &original, uint32_t start, int32_t end) = 0; + + virtual bool IsSameParam(SampleParam *param) = 0; + +protected: + std::string algorithm_; +}; +} +#endif //MINDSTUDIO_BOARD_PLUGINS_MINDSTUDIO_INSIGHT_PLUGINS_SCALAR_SERVER_SRC_SAMPLER_SAMPLERBASE_H_ diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/SamplerFactory.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/SamplerFactory.h new file mode 100644 index 0000000000000000000000000000000000000000..0f30b5564d9944191e2140f6f3dc5fa45bc1d11b --- /dev/null +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/Sampler/SamplerFactory.h @@ -0,0 +1,46 @@ +/* + * Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. + */ +#ifndef MINDSTUDIO_BOARD_SAMPLERFACTORY_H +#define MINDSTUDIO_BOARD_SAMPLERFACTORY_H + +#include "SamplerBase.h" +#include "IRLowPassSampler.h" +#include +#include +#include +#include +#include + +namespace Insight::Scalar::Sample { +class SampleFactory { +public: + static SampleFactory &Instance() + { + static SampleFactory instance; + return instance; + } + + SampleFactory() + { + sampleMap_.emplace("smoothing", []() { + return std::make_unique(); + }); + } + + std::unique_ptr GetSampler(std::string_view algorithm) + { + auto it = sampleMap_.find(algorithm); + if (it == sampleMap_.end()) { + return nullptr; + } + auto func = it->second; + return func(); + } + +private: + std::unordered_map()>> sampleMap_{}; +}; + +} +#endif //MINDSTUDIO_BOARD_SAMPLERFACTORY_H diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/Util/ScalaryProtocolUtil.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/Util/ScalaryProtocolUtil.h index 09ca25ffbfdd296e65fea3d1fd39a70b75bbcce9..0424d2bd2d8cc02cfe54036a9c235f43eaac3dab 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/Util/ScalaryProtocolUtil.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/Util/ScalaryProtocolUtil.h @@ -36,6 +36,10 @@ struct SingleGraphReqInfo { struct GetScalarDataRequest { std::vector data_; }; + +struct GetParseStateReq { + std::string projectName_; +}; using json = rapidjson::Value; using document_t = rapidjson::Document; diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/FileParser.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/FileParser.cpp index 24350509ddaedd6e5a3f829811fe5c663c15cb37..e48294948bf8755db95f78844cd2ea74b41f11b3 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/FileParser.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/FileParser.cpp @@ -10,20 +10,7 @@ using namespace Insight::Scalar; using namespace Insight; bool FileParser::CheckFilePathVaild(const std::string &filePath) { - if (filePath.empty()) { - LOG(LogRank::Error) << "File path is empty"; - return false; - } - if (!fs::exists(filePath)) { - LOG(LogRank::Error) << "File path does not exist"; - return false; - } - auto readPermission = fs::status(filePath).permissions() & fs::perms::owner_read; - if (readPermission == fs::perms::none) { - LOG(LogRank::Error) << "File not permit to read"; - return false; - } - return true; + } std::ifstream FileParser::OpenFileSafe(const std::string &filePath) { @@ -37,16 +24,18 @@ std::ifstream FileParser::OpenFileSafe(const std::string &filePath) { return file; } -std::map > FileParser::ParserData(const std::string &filePath, uint64_t &offset) { - std::map > data; +bool FileParser::ParserData(const std::string &filePath, uint64_t &offset) +{ std::ifstream file = OpenFileSafe(filePath); if (!file.is_open()) { LOG(LogRank::Warning) << "Parse data faild, open file error"; - return data; + return true; } file.seekg(static_cast(offset), std::ios::beg); std::string recordStr; + bool flag = true; while (file && ReadRecord(recordStr, file)) { + std::map > data; if (!ParseRecordToScalar(std::move(recordStr), data)) { break; } @@ -54,8 +43,15 @@ std::map > FileParser::ParserData(const st if (index != -1) { offset = index; } + if (data.empty()) { + continue; + } + flag = false; + for (auto &[tag, points]: data) { + updater_.UpdateData(filePath, tag, std::move(points)); + } } - return data; + return flag; } bool FileParser::ReadCheckSumRecord(std::ifstream &input, std::vector &buffer, size_t size) { diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/FileParser.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/FileParser.h index 3952ac4c7427d1917d9ac5695ece8f32f20dbbe9..8a14408736b9b1781f2b9842161bd4b8942959ae 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/FileParser.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/FileParser.h @@ -9,6 +9,7 @@ #include #include #include +#include #include "defs/ConceptDefs.h" namespace Insight::Scalar::Parser { @@ -17,7 +18,8 @@ namespace Insight::Scalar::Parser { */ class FileParser { public: - FileParser() : type_(ParseDataType::Unknown) {} + FileParser() : type_(ParseDataType::Unknown), updater_() + {} /** * @brief open file stream safely @@ -27,18 +29,48 @@ public: std::ifstream OpenFileSafe(const std::string &filePath); /** - * + * @brief parse data top function * @param filePath * @param[in/out] offset file read offset, update after parsed * @return */ - std::map> ParserData(const std::string &filePath, - uint64_t &offset); + bool ParserData(const std::string &filePath, uint64_t &offset); + /** + * @brief read one record form file + * @param recordStr + * @param input + * @return + */ virtual bool ReadRecord(std::string &recordStr, std::ifstream &input) = 0; + /** + * @brief parse the record to scalar data + * @param record + * @param res + * @return + */ virtual bool ParseRecordToScalar(std::string &&record, std::map> &res) = 0; + + /** + * @brief update data to manager + * @param filePath + * @param Tag + * @param data + */ + inline void UpdateTagData(const std::string &filePath, const std::string &tag, std::vector &&data) + { + updater_.UpdateData(filePath, tag, std::move(data)); + }; + + /** + * @brief read record implement + * @param input + * @param buffer + * @param size + * @return + */ static bool ReadCheckSumRecord(std::ifstream &input, std::vector &buffer, size_t size); virtual ~FileParser() = default; @@ -48,6 +80,7 @@ private: public: ParseDataType type_; + DataUpdateImp updater_; }; } diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/LogTextParser.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/LogTextParser.cpp index 5573bf71e91e3c89a915770eca6a0f7ce085e8e9..30378fe8b488f2b34c53cc33c344c2bd6c3ed90a 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/LogTextParser.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/LogTextParser.cpp @@ -15,13 +15,13 @@ bool LogTextParser::ReadRecord(std::string &recordStr, std::ifstream &input) { bool LogTextParser::ParseRecordToScalar(std::string &&record, std::map> &res) { if (record.empty()) { - return false; + return true; } // parse step information int64_t step = 0; constexpr size_t MATCH_INDEX = 2; constexpr size_t MATCH_INDEX_4 = 4; - const auto stepRegex = std::regex(R"((step:\[\s(\d+)/) | (iteration\s{5}(\d+)/))"); + const auto stepRegex = std::regex(R"((step:\[\s*(\d+)/) | (iteration\s+(\d+)/))"); if (std::smatch smatch; std::regex_search(record, smatch, stepRegex)) { if (smatch.size() > MATCH_INDEX && smatch[MATCH_INDEX].matched) { step = std::stoll(smatch[MATCH_INDEX].str()); @@ -33,14 +33,23 @@ bool LogTextParser::ParseRecordToScalar(std::string &&record, std::map MATCH_INDEX && smatch[MATCH_INDEX].matched) { - value = std::stof(smatch[MATCH_INDEX].str()); + if (smatch.size() > 1 && smatch[1].matched) { + value = std::stof(smatch[1].str()); + res[TEXT_DEFAULT_TAG_LOSS.data()].emplace_back(step, value); } } else { - return true; + return true; + } + const auto normalRegex = std::regex(R"(global_norm:\s*\[([-+]?\d*\.\d+|\d+))"); + if (std::smatch smatch; std::regex_search(record, smatch, normalRegex)) { + if (smatch.size() > 1 && smatch[1].matched) { + value = std::stof(smatch[1].str()); + res[TEXT_DEFAULT_TAG_GLOBAL_NORM.data()].emplace_back(step, value); + } + } else { + return true; } - res[TEXT_DEFAULT_TAG.data()].emplace_back(step, value); return true; } diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/LogTextParser.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/LogTextParser.h index 639af0eb33e76b82ae6cbc9296cb698eb4242e9a..8f0fa6b4763e5609e28fcf543278f844f42feebc 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/LogTextParser.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/LogTextParser.h @@ -7,18 +7,19 @@ #include "FileParser.h" namespace Insight::Scalar::Parser { -constexpr std::string_view TEXT_DEFAULT_TAG = "Loss"; - +constexpr std::string_view TEXT_DEFAULT_TAG_LOSS = "Loss"; +constexpr std::string_view TEXT_DEFAULT_TAG_GLOBAL_NORM = "global_norm"; class LogTextParser : public FileParser { public: LogTextParser() { type_ = ParseDataType::TEXT_LOG; } + bool ReadRecord(std::string &recordStr, std::ifstream &input) override; + bool ParseRecordToScalar(std::string &&record, std::map> &res) override; - ~LogTextParser() override = default; -private: + ~LogTextParser() override = default; }; } diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/ParserFactory.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/ParserFactory.h index 76130c8feb8fdddb11c15e3ba3b0b0e7e9ef2cb7..4e9aaca4360ebe710f68113cae7d8a8c05675a32 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/ParserFactory.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/ParserFactory.h @@ -34,7 +34,7 @@ private: static inline std::map > parsers_ = { {ParseDataType::TF_EVENT, std::make_shared()}, - {ParseDataType::MindSpore_Summary, std::make_shared()}, + {ParseDataType::MINDSPORE_SUMMARY, std::make_shared()}, {ParseDataType::TEXT_LOG, std::make_shared()}, {ParseDataType::Unknown, nullptr} }; diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/SummaryParser.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/SummaryParser.h index 4b18e56ca75bb765684bc3ebec6374306aa11273..71fe39d3cfa7314f78b6a10434f1422dd6a33c47 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/SummaryParser.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/parser/SummaryParser.h @@ -11,11 +11,13 @@ namespace Insight::Scalar::Parser { class SummaryParser : public FileParser { public: SummaryParser() { - type_ = ParseDataType::MindSpore_Summary; + type_ = ParseDataType::MINDSPORE_SUMMARY; } bool ReadRecord(std::string &eventStr, std::ifstream &input) override; + bool ParseRecordToScalar(std::string &&record, std::map> &res) override; + ~SummaryParser() override = default; private: diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetAllGraphHandler.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetAllGraphHandler.cpp index 4b4f0dcf7c38041e58e45f453cc4daa2f73daf6f..18fe82b519d64e771bcfd0bb3b3621318459d9b9 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetAllGraphHandler.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetAllGraphHandler.cpp @@ -12,6 +12,7 @@ namespace fs = std::experimental::filesystem; #endif #include "ScalarVisuallyServer.h" #include "Util/ScalaryProtocolUtil.h" + using namespace Insight::Scalar; using namespace Insight::Scalar::Protocol; diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetScalarDataHandler.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetScalarDataHandler.cpp index 2806bc9847e0729389de9f51fe032c902e69bac0..e5055c9357227a69ad0478d146736b083f1ee97c 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetScalarDataHandler.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetScalarDataHandler.cpp @@ -53,7 +53,7 @@ bool ScalarVisuallyGetScalarDataHandler::CheckParamValid(const document_t &reque if (!item.HasMember("sampleOffset") || !item["sampleOffset"].IsUint64()) { return false; } - if (!item.HasMember("sampleWeight") || !item["sampleWeight"].IsFloat()) { + if (!item.HasMember("sampleWeight") || !item["sampleWeight"].IsNumber()) { return false; } return true; diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetScalarDataHandler.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetScalarDataHandler.h index e7b8251a20a5a36ca2893ffd21e621ff97eb0ad3..09756c18de81e1e333bf0052afb31c6d7a5febc2 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetScalarDataHandler.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyGetScalarDataHandler.h @@ -17,11 +17,13 @@ using namespace Scalar::GraphOp; class ScalarVisuallyGetScalarDataHandler : public PostHandler { public: bool run(std::string_view data, std::string &resultStr) override; + static bool CheckParamValid(const document_t &request); private: static void SetResponse(std::vector &&responseData, std::string &resultStr); + static inline ErrCode ParseRequestFromJson(std::string_view data, GetScalarDataRequest &request, std::string &errMsg); diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyImportFileHandler.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyImportFileHandler.cpp index cd6bdee35872fc5ab94cd2b8b9595231b5bea1c0..5d6200c3a8a7f6a0f928bc9f5b00bf4228618fa0 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyImportFileHandler.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyImportFileHandler.cpp @@ -5,6 +5,7 @@ #include #include #include + #ifdef _WIN32 #include namespace fs = std::filesystem; @@ -32,21 +33,12 @@ bool ScalarVisuallyImportFileHandler::run(std::string_view data, std::string &re ScalarVisuallyServer::Instance().Reset(); } auto &pathList = request.pathList_; - + std::string projectName = ScalarVisuallyServer::GetProjectName(); // get all file which need to be import std::vector importFiles = GetImportFiles(pathList); - // delete the invalid path - const auto end = std::remove_if(pathList.begin(), pathList.end(), [](const auto &path) { - return ScalarVisuallyImportFileHandler::PathInvalid(path); - }); - pathList.erase(end); - // import file, record the tags of parsed result - std::set graphTags; - for (auto &file: importFiles) { - HandleImportFile(std::move(file), graphTags); - } - SetResponse(graphTags, resultStr); + ScalarVisuallyServer::Instance().AddParseTask(projectName, importFiles); + SetResponse(projectName, resultStr); return true; } @@ -62,62 +54,29 @@ bool ScalarVisuallyImportFileHandler::PathInvalid(std::string_view path) { return false; } -bool ScalarVisuallyImportFileHandler::HandleImportFile(std::string &&path, std::set &graphTags) { - ScalarVisuallyServer &instance = ScalarVisuallyServer::Instance(); - if (instance.IsFileImported(path)) { - LOG(LogRank::Info) << "File already imported before"; - return true; - } - const std::shared_ptr fileInfo = instance.AddFile(path); - const auto parser = instance.GetFileParser(fileInfo->parseDataType_); - if (!parser) { - return false; - } - std::map > parsedData = parser->ParserData(path, fileInfo->offSet_); - if (parsedData.empty()) { - LOG(LogRank::Error) << "File not contains data, file=" << path; - return false; - } - for (auto &[tag, data]: parsedData) { - graphTags.insert(tag); - ScalarVisuallyServer::Instance().UpdateGraph(tag, path, std::move(data)); - } - ScalarVisuallyServer::Instance().AddFileWatch({path}); - return true; -} - std::vector ScalarVisuallyImportFileHandler::GetImportFiles(std::vector &pathList) { std::vector res; for (const auto &path: pathList) { - if (!fs::is_directory(path) && ScalarVisuallyServer::IsFileSupported(path)) { - res.emplace_back(path); + try { + if (!fs::is_directory(path) && ScalarVisuallyServer::IsFileSupported(path)) { + res.emplace_back(path); + } + constexpr uint32_t MAX_DEPTH = 7; + RecursiveScanFolder(path, res, MAX_DEPTH); + } catch (const fs::filesystem_error &e) { + LOG(LogRank::Error) << "Cause filesystem error when import, e =" << e.what(); + continue; } - constexpr uint32_t MAX_DEPTH = 7; - RecursiveScanFolder(path, res, MAX_DEPTH); } return res; } -void ScalarVisuallyImportFileHandler::SetResponse(std::set &graphTags, std::string &resultStr) { +void ScalarVisuallyImportFileHandler::SetResponse(const std::string &projectName, std::string &resultStr) +{ document_t document = ParseJsonToStr(resultStr); auto &allocator = document.GetAllocator(); json &body = document["body"]; - json data(rapidjson::kArrayType); - for (auto &tag: graphTags) { - std::vector dataFiles = ScalarVisuallyServer::Instance().GetGraphInfo(tag); - json graph(rapidjson::kObjectType); - AddJsonMember(graph, "tag", tag, allocator); - json fileList(rapidjson::kArrayType); - for (const std::string &dataFile: dataFiles) { - json file(rapidjson::kObjectType); - AddJsonMember(file, "path", dataFile, allocator); - AddJsonMember(file, "name", GetReadableFileName(dataFile), allocator); - fileList.PushBack(file, allocator); - } - AddJsonMember(graph, "fileList", fileList, allocator); - data.PushBack(graph, allocator); - } - AddJsonMember(body, "data", data, allocator); + AddJsonMember(body, "projectName", projectName, allocator); resultStr = DumpJsonToStr(document); } @@ -181,6 +140,10 @@ void ScalarVisuallyImportFileHandler::RecursiveScanFolder(const std::string &pat if (curDepth == maxDepth) { continue; } + if (auto per = fs::status(curPath).permissions(); (per & fs::perms::owner_read) == fs::perms::none) { + LOG(LogRank::Error) << "Cur path has no read permission"; + continue; + } for (const auto &entry: fs::directory_iterator(curPath)) { if (fs::is_directory(entry)) { searchQueue.emplace(entry.path().string(), curDepth + 1); diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyImportFileHandler.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyImportFileHandler.h index 3f2d84f4aac73ac934933493a1c310e3b3d9a21d..495f3b6b2f080662c1151466fab39f7d09dc7d5b 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyImportFileHandler.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/Handler/ScalarVisuallyImportFileHandler.h @@ -24,12 +24,17 @@ public: static bool CheckParamValid(const document_t &request); private: + static ErrCode ParseReqeustFromJson(std::string_view data, ImportFileRequest &request, std::string &errMsg); + static bool PathInvalid(std::string_view); + static std::vector GetImportFiles(std::vector &pathList); - static bool HandleImportFile(std::string &&path, std::set &graphTags); - static void SetResponse(std::set &graphTags, std::string &resultStr); + + static void SetResponse(const std::string &projectName, std::string &resultStr); + static void AddFileWatch(const std::string &path); + static void RecursiveScanFolder(const std::string &path, std::vector &fileList, int maxDepth); diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/ScalarVisuallyServer.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/ScalarVisuallyServer.cpp index 31cf15a36f6eea00494a0dfeeceea90d83873dc0..523c5819a34551db50f348765821fa0d60c89d0c 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/ScalarVisuallyServer.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/ScalarVisuallyServer.cpp @@ -3,6 +3,8 @@ */ #include "ScalarVisuallyServer.h" #include +#include +#include #include "Logger.h" #include "FileManager/FileInfoManager.h" @@ -21,7 +23,7 @@ bool ScalarVisuallyServer::IsFileImported(std::string_view path) { void ScalarVisuallyServer::OnFileDataUpdate(std::string &&dir, std::string &&fileName) { std::string filePath = dir + "/" + fileName; - std::shared_ptr file = fileManager_.GetFileInfo(filePath); + std::shared_ptr file = fileManager_.GetFileInfo(filePath); if (file == nullptr) { return; } @@ -29,11 +31,9 @@ void ScalarVisuallyServer::OnFileDataUpdate(std::string &&dir, std::string &&fil if (!parser) { return; } - std::map> data = parser->ParserData(filePath, file->offSet_); - for (auto &[tag, points]: data) { - graphManager_.UpdateGraphData(tag, filePath, std::move(points)); - } - if (!data.empty() && file->empty_) { + bool empty = parser->ParserData(filePath, file->offSet_); + + if (!empty && file->empty_) { ScalarVisuallyServer::Instance().OnFileCreate(std::move(dir), std::move(fileName)); file->empty_ = false; } @@ -43,7 +43,8 @@ bool ScalarVisuallyServer::IsFileSupported(std::string_view path) { return FileInfo::FileInfoManager::IsFileSupported(path); } -std::shared_ptr ScalarVisuallyServer::AddFile(const std::string &file) { +std::shared_ptr ScalarVisuallyServer::AddFile(const std::string &file) +{ ParseDataType type = FileInfo::FileInfoManager::GetFileType(file); return fileManager_.AddFile(file, type); } @@ -94,3 +95,107 @@ std::unordered_map> ScalarVisuallyServer::Get return fileManager_.GetCreatedFileGroupByDir(); } +std::string ScalarVisuallyServer::GetProjectName() +{ + static uint64_t projectId = 0; + return std::to_string(projectId++); +} + +bool ScalarVisuallyServer::AddParseTask(const std::string &projectName, std::vector fileList) +{ + std::vector> fileInfos; + std::for_each(fileList.begin(), fileList.end(), [&fileInfos](const std::string &filePath) { + if (ScalarVisuallyServer::Instance().IsFileImported(filePath)) { + return; + } + auto fileInfo = ScalarVisuallyServer::Instance().AddFile(filePath); + fileInfos.push_back(fileInfo); + }); + // calculate total data size for calculate percent + ParseState &s = parseStateMap_[projectName]; + s.dataSize_ = std::accumulate(fileInfos.begin(), fileInfos.end(), static_cast(0), + [](int acc, const std::shared_ptr &fileInfo) { + return acc + fs::file_size(fileInfo->filePath_); + }); + + // add parsed thread + std::for_each(fileInfos.begin(), fileInfos.end(), + [this, &projectName](const std::shared_ptr &fileInfo) { + auto parser = this->GetFileParser(fileInfo->parseDataType_); + if (parser == nullptr) { + return; + } + ParseState &state = parseStateMap_[projectName]; + auto future = threadPool_->enqueue([this, parser, fileInfo]() { + return parser->ParserData(fileInfo->filePath_, fileInfo->offSet_); + }); + state.parsedResults_.emplace_back(std::move(future)); + }); + return true; +} + +void ScalarVisuallyServer::parseWatcher() +{ + for (;;) { + // lock here, in case other thread delete parseState while use iter list + std::shared_lock lock(parseMutex_); + for (auto &[projectName, state]: this->parseStateMap_) { + // reduce + if (state.IsFinish()) { + continue; + } + // check future get value + if (std::all_of(state.parsedResults_.begin(), state.parsedResults_.end(), [](auto &future) { + return future.wait_for(std::chrono::seconds(0)) == std::future_status::ready; + })) { + state.finish_ = true; + state.percent_ = 100; + continue; + } + // update percent + uint64_t parsedDataSize = 0; + std::for_each(state.fileInfos_.begin(), state.fileInfos_.end(), [&parsedDataSize](const auto &fileInfo) { + parsedDataSize += fileInfo->offSet_; + }); + state.percent_ = std::min(static_cast(100), + static_cast((parsedDataSize / state.dataSize_) * 100 )); + + } + } +} + +std::pair ScalarVisuallyServer::GetProjectParseStatus(const std::string &projectName) +{ + std::shared_lock lock(parseMutex_); + ParseState &s = parseStateMap_[projectName]; + return {s.IsFinish(), s.percent_}; +} + +bool ScalarVisuallyServer::IsIncremental(const std::string &filePath, const std::string &tag) +{ + // check whether new tag + auto graph = graphManager_.GetGraph(tag); + if (graph == nullptr) { + return true; + } + // check whether new file + if (!graph->InnerFile(filePath)) { + return true; + } + return false; +} + +void ScalarVisuallyServer::AddIncremental(const std::string &filePath, const std::string &tag) +{ + std::unique_lock lock(incrementalMutex_); + incrementalTag_[tag].insert(filePath); +} + +std::unordered_map> ScalarVisuallyServer::GetIncremental() +{ + std::unordered_map> res; + std::unique_lock lock(incrementalMutex_); + res = std::move(incrementalTag_); // stolen the data first, + incrementalTag_ = std::unordered_map>(); + return res; +} diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/ScalarVisuallyServer.h b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/ScalarVisuallyServer.h index b5cacb372fda10ec661eb9a1767f512fc1aae66f..c69b4f358e712fd39b2496a285c5fa969a1950bd 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/ScalarVisuallyServer.h +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/plugin/ScalarVisuallyServer.h @@ -6,6 +6,8 @@ #include #include +#include +#include #include "defs/ConceptDefs.h" #include "FileManager/FileInfoManager.h" #include "GraphManager/GraphManager.h" @@ -13,12 +15,27 @@ #include "ParserFactory.h" #include "Logger.h" #include "Util/ScalaryProtocolUtil.h" +#include "Util/ThreadPool.h" using namespace Insight; namespace Insight::Scalar { using namespace GraphOp; using namespace Protocol; +struct ParseState { + std::vector> parsedResults_; + std::vector> fileInfos_; + uint64_t dataSize_; + uint32_t percent_{0}; + bool finish_{false}; + + void UpdatePercent() + {}; + + bool IsFinish() + { return finish_; } +}; + class ScalarVisuallyServer { public: static ScalarVisuallyServer &Instance() { @@ -28,7 +45,7 @@ public: ~ScalarVisuallyServer() = default; // 文件增删改查 - std::shared_ptr AddFile(const std::string &file); + std::shared_ptr AddFile(const std::string &file); bool IsFileImported(std::string_view path); void AddFileWatch(const std::vector &fileList); bool IsFileWatched(std::string &&path); @@ -36,6 +53,18 @@ public: void GetFileTags(std::set &tags, std::string &path); std::unordered_map > GetCreatedFileGroupByDir(); + // Parse file func + bool AddParseTask(const std::string &projectName, std::vector fileList); + + void parseWatcher(); + + std::pair GetProjectParseStatus(const std::string &projectName); + + bool IsIncremental(const std::string &filePath, const std::string &tag); + + void AddIncremental(const std::string &filePath, const std::string &tag); + + std::unordered_map> GetIncremental(); // 文件更新 void OnFileDataUpdate(std::string &&dir, std::string &&fileName); @@ -48,20 +77,31 @@ public: std::unordered_map > GetAllGraphInfo(); void UpdateGraph(const std::string &tag, const std::string &path, std::vector &&data); + static std::string GetProjectName(); void Reset(); private: - ScalarVisuallyServer() { + ScalarVisuallyServer() noexcept + { fileWatcher_ = FileWatch::FileWatcherFactory::GetFileWatcher(); if (!fileWatcher_) { LOG(LogRank::Warning) << "This platform not support file realtime watch"; return; } fileWatcher_->Init(); + threadPool_ = std::make_shared( + std::min(std::thread::hardware_concurrency(), static_cast(16))); }; + Scalar::FileInfo::FileInfoManager fileManager_{}; GraphManager graphManager_{}; std::unique_ptr fileWatcher_{nullptr}; + std::shared_ptr threadPool_{nullptr}; + std::unordered_map parseStateMap_; + std::unordered_map> incrementalTag_; // tag - [filePath] + std::mutex incrementalMutex_; + std::shared_mutex parseMutex_; + }; } diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/test/GraphTest.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/test/GraphTest.cpp index da68e4f158bd8a241f4fa9092b125ec8c68fc2f5..3d290d0508f9398b0241fd074e36ddaf6fa1de7d 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/test/GraphTest.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/test/GraphTest.cpp @@ -3,6 +3,7 @@ */ #include "gtest/gtest.h" #include "GraphManager/GraphManager.h" + using namespace Insight::Scalar; using namespace Insight::Scalar::GraphOp; diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/test/ParserTest.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/test/ParserTest.cpp index 5f76cfdc5a626994de7421c4104657e73a14be2c..55c93677e5bb36a65bfa6b7a5a36f0e8d5653500 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/test/ParserTest.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/test/ParserTest.cpp @@ -1,10 +1,11 @@ +/* +* Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. +*/ #include "gtest/gtest.h" #include "ParserFactory.h" #include "proto/event.pb.h" #include "defs/ConceptDefs.h" -/* -* Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. -*/ + using namespace Insight::Scalar::Parser; using namespace Insight::Scalar; diff --git a/plugins/mindstudio-insight-plugins/Scalar/server/src/test/PluginTest.cpp b/plugins/mindstudio-insight-plugins/Scalar/server/src/test/PluginTest.cpp index c53ab65385617338eda6c23fd0dab49b0353f74b..b36507f67901785d160cc336a2b6b83f5c76b37e 100644 --- a/plugins/mindstudio-insight-plugins/Scalar/server/src/test/PluginTest.cpp +++ b/plugins/mindstudio-insight-plugins/Scalar/server/src/test/PluginTest.cpp @@ -6,7 +6,9 @@ #include namespace fs = std::filesystem; #else + #include + namespace fs = std::experimental::filesystem; #endif #include "gtest/gtest.h" diff --git a/plugins/mindstudio-insight-plugins/plugin_core/CMakeLists.txt b/plugins/mindstudio-insight-plugins/plugin_core/CMakeLists.txt index b246fb4429391224f761996b9e39fce4d2ad6fb2..8e53d91902f1fa64a0e4fba16b6e9c028e96e37a 100644 --- a/plugins/mindstudio-insight-plugins/plugin_core/CMakeLists.txt +++ b/plugins/mindstudio-insight-plugins/plugin_core/CMakeLists.txt @@ -1,16 +1,18 @@ project(msinsight) - +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_C_STANDARD 11) aux_source_directory(./src CORE_SRC) - +set(LIBRARY_OUTPUT_PATH ${PROJECT_ROOT_DIR}/output/lib) add_library(${PROJECT_NAME} SHARED ${CORE_SRC}) target_include_directories(${PROJECT_NAME} PUBLIC ./include) target_include_directories(${PROJECT_NAME} PRIVATE ./src) if (CMAKE_SYSTEM_NAME STREQUAL "Linux") - target_link_libraries(${PROJECT_NAME} PRIVATE dl) - target_link_libraries(${PROJECT_NAME} PRIVATE stdc++fs) + target_link_libraries(${PROJECT_NAME} PUBLIC dl) + target_link_libraries(${PROJECT_NAME} PUBLIC stdc++fs) endif () if (${CMAKE_BUILD_TYPE} MATCHES "Debug") target_compile_options(${PROJECT_NAME} PRIVATE -g -O0) endif () + diff --git a/plugins/mindstudio-insight-plugins/plugin_core/cmake/mind_expression.cmake b/plugins/mindstudio-insight-plugins/plugin_core/cmake/mind_expression.cmake index 7e1097011b20ff20b709ab685782c4a47da0d651..65bca33e8528a381846716048b49fe7760aa1bce 100644 --- a/plugins/mindstudio-insight-plugins/plugin_core/cmake/mind_expression.cmake +++ b/plugins/mindstudio-insight-plugins/plugin_core/cmake/mind_expression.cmake @@ -15,7 +15,7 @@ endif() set(TOP_DIR ${CMAKE_SOURCE_DIR}) set(ENABLE_GITEE ON) -include(${CMAKE_CURRENT_SOURCE_DIR}/plugin_core/cmake/options.cmake) +#include(${CMAKE_CURRENT_SOURCE_DIR}/plugin_core/cmake/options.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin_core/cmake/utils.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin_core/cmake/external_libs/protobuf.cmake) include(${CMAKE_CURRENT_SOURCE_DIR}/plugin_core/cmake/external_libs/json.cmake) diff --git a/plugins/mindstudio-insight-plugins/plugin_core/src/PluginsManager.cpp b/plugins/mindstudio-insight-plugins/plugin_core/src/PluginsManager.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bf03b6850e71f4a191c92097160e08592f804b6f --- /dev/null +++ b/plugins/mindstudio-insight-plugins/plugin_core/src/PluginsManager.cpp @@ -0,0 +1,60 @@ +#include "PluginsManager.h" +#include + +using namespace Dic::Core; +#ifdef _WIN32 +#include +namespace fs = std::filesystem; +const static std::string EXT = ".dll"; +#else + +#include +#include + +namespace fs = std::experimental::filesystem; +const static std::string EXT = ".so"; +#endif + +PluginsManager &PluginsManager::Instance() +{ + static PluginsManager instance; + return instance; +} + +bool PluginsManager::RegisterPlugin(std::unique_ptr plugin) +{ + pluginsMap_.emplace(plugin->GetPluginName(), std::move(plugin)); + return true; +} + +void PluginsManager::LoadPlugins() +{ + auto pluginsDir = PLUGINS_DIR; + if (!fs::exists(pluginsDir)) { + return; + } + for (auto &dir: fs::directory_iterator(pluginsDir)) { + if (!fs::is_directory(dir)) { + continue; + } + for (auto &file: fs::directory_iterator(dir)) { + if (!fs::is_directory(file) && file.path().extension().string() == EXT) { +#ifdef _WIN32 +#else + dlopen(file.path().string().c_str(), RTLD_LAZY); +#endif + } + } + } + +} + +std::map> &PluginsManager::GetAllPlugins() +{ + return pluginsMap_; +} + +PluginRegister::PluginRegister(std::unique_ptr plugin) +{ + PluginsManager::Instance().RegisterPlugin(std::move(plugin)); +} diff --git a/plugins/mindstudio-insight-plugins/proto/CMakeLists.txt b/plugins/mindstudio-insight-plugins/proto/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..0644e2dbb4155f40b4331153fbfed9ad3acdaf17 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/CMakeLists.txt @@ -0,0 +1,6 @@ +project(PROTO_GENERATE) +message(STATUS ${PROJECT_ROOT_DIR}) +file(GLOB_RECURSE PROTO_FILE "${PROJECT_ROOT_DIR}/proto/*.proto") +message(.) +ms_protobuf_generate(${PROJECT_NAME} ${CMAKE_CURRENT_SOURCE_DIR} PROTO_SRC PROTO_H ${PROTO_FILE}) + diff --git a/plugins/mindstudio-insight-plugins/proto/event.proto b/plugins/mindstudio-insight-plugins/proto/event.proto new file mode 100644 index 0000000000000000000000000000000000000000..f135df01e22caeaafe7fc9e9c32efb98f017fc40 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/event.proto @@ -0,0 +1,141 @@ +syntax = "proto3"; + +package tensorboard; + +import "summary.proto"; + +option cc_enable_arenas = true; +option java_outer_classname = "EventProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.util"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/util/event_go_proto"; + +// Protocol buffer representing an event that happened during +// the execution of a Brain model. +message Event { + // Timestamp of the event. + double wall_time = 1; + + // Global step of the event. + int64 step = 2; + + oneof what { + // An event file was started, with the specified version. + // This is use to identify the contents of the record IO files + // easily. Current version is "brain.Event:2". All versions + // start with "brain.Event:". + string file_version = 3; + // An encoded version of a GraphDef. + bytes graph_def = 4; + // A summary was generated. + Summary summary = 5; + // The user output a log message. This was theoretically used by the defunct + // tensorboard_logging module, which has since been removed; this field is + // now deprecated and should not be used. + LogMessage log_message = 6 [deprecated = true]; + // The state of the session which can be used for restarting after crashes. + SessionLog session_log = 7; + // The metadata returned by running a session.run() call. + TaggedRunMetadata tagged_run_metadata = 8; + // An encoded version of a MetaGraphDef. + bytes meta_graph_def = 9; + } + + // Information of the source that writes the events, this is only logged in + // the very first event along with the `file_version` field. + SourceMetadata source_metadata = 10; +} + +// Holds the information of the source that writes the events. +message SourceMetadata { + // Low level name of the summary writer, such as + // `tensorflow.core.util.events_writer`. + string writer = 1; +} + +// Protocol buffer used for logging messages to the events file. +// +// This was theoretically used by the defunct tensorboard_logging module, which +// has been removed; this message is now deprecated and should not be used. +message LogMessage { + option deprecated = true; + enum Level { + option deprecated = true; + UNKNOWN = 0; + // Note: The logging level 10 cannot be named DEBUG. Some software + // projects compile their C/C++ code with -DDEBUG in debug builds. So the + // C++ code generated from this file should not have an identifier named + // DEBUG. + DEBUGGING = 10; + INFO = 20; + WARN = 30; + ERROR = 40; + FATAL = 50; + } + Level level = 1; + string message = 2; +} + +// Protocol buffer used for logging session state. +message SessionLog { + enum SessionStatus { + STATUS_UNSPECIFIED = 0; + START = 1; + STOP = 2; + CHECKPOINT = 3; + } + + SessionStatus status = 1; + // This checkpoint_path contains both the path and filename. + string checkpoint_path = 2; + string msg = 3; +} + +// For logging the metadata output for a single session.run() call. +message TaggedRunMetadata { + // Tag name associated with this metadata. + string tag = 1; + // Byte-encoded version of the `RunMetadata` proto in order to allow lazy + // deserialization. + bytes run_metadata = 2; +} + +// Worker heartbeat messages. Support for these operations is currently +// internal and expected to change. + +// Current health status of a worker. +enum WorkerHealth { + OK = 0; // By default a worker is healthy. + RECEIVED_SHUTDOWN_SIGNAL = 1; + INTERNAL_ERROR = 2; + SHUTTING_DOWN = 3; // Worker has been instructed to shutdown after a timeout. +} + +// Indicates the behavior of the worker when an internal error or shutdown +// signal is received. +enum WorkerShutdownMode { + DEFAULT = 0; + NOT_CONFIGURED = 1; + WAIT_FOR_COORDINATOR = 2; + SHUTDOWN_AFTER_TIMEOUT = 3; +} + +message WatchdogConfig { + int64 timeout_ms = 1; +} + +message RequestedExitCode { + int32 exit_code = 1; +} + +message WorkerHeartbeatRequest { + WorkerShutdownMode shutdown_mode = 1; + WatchdogConfig watchdog_config = 2; + RequestedExitCode exit_code = 3; +} + +message WorkerHeartbeatResponse { + WorkerHealth health_status = 1; + repeated Event worker_log = 2; + string hostname = 3; +} diff --git a/plugins/mindstudio-insight-plugins/proto/histogram.proto b/plugins/mindstudio-insight-plugins/proto/histogram.proto new file mode 100644 index 0000000000000000000000000000000000000000..568a90b73a1adee038a6eaf09f759f413f97a3f4 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/histogram.proto @@ -0,0 +1,26 @@ +syntax = "proto3"; + +package tensorboard; + +option cc_enable_arenas = true; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/google/tsl/tsl/go/core/protobuf/summary_go_proto"; + +// Serialization format for histogram module in +// tsl/lib/histogram/histogram.h +message HistogramProto { + double min = 1; + double max = 2; + double num = 3; + double sum = 4; + double sum_squares = 5; + + // Parallel arrays encoding the bucket boundaries and the bucket values. + // bucket(i) is the count for the bucket i. The range for + // a bucket is: + // i == 0: -DBL_MAX .. bucket_limit(0) + // i != 0: bucket_limit(i-1) .. bucket_limit(i) + repeated double bucket_limit = 6 [packed = true]; + repeated double bucket = 7 [packed = true]; +} diff --git a/plugins/mindstudio-insight-plugins/proto/mindspore_anf_ir.proto b/plugins/mindstudio-insight-plugins/proto/mindspore_anf_ir.proto new file mode 100644 index 0000000000000000000000000000000000000000..f221e6d7f75358276382287b6741050bf51cb13f --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/mindspore_anf_ir.proto @@ -0,0 +1,353 @@ +/** + * Copyright 2019 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package mindspore.irpb; + + +// Versioning +enum Version { + // unknown version + UNKNOWWN_VERSION = 0; + + // Initial version (IR VERSION 1), published on Sep 23, 2019 + IR_VERSION = 0x0000000000000001; +} + +// Data type definition +enum DataType { + DT_UNDEFINED = 0; + // Basic types. + DT_BOOL = 1; // bool + + DT_INT8 = 2; // int8_t + DT_INT16 = 3; // int16_t + DT_INT32 = 4; // int32_t + DT_INT64 = 5; // int64_t + + DT_UINT8 = 6; // uint8_t + DT_UINT16 = 7; // uint16_t + DT_UINT32 = 8; // uint32_t + DT_UINT64 = 9; // uint64_t + + DT_FLOAT16 = 10; // float 16 + DT_FLOAT32 = 11; // float 32 + DT_FLOAT64 = 12; // float 64 + + DT_STRING = 13; // string + DT_TENSOR = 14; // tensor + DT_GRAPH = 15; // graph + + // list type + DT_BOOLS = 16; // list of bool + + DT_INTS8 = 17; // list of int8_t + DT_INTS16 = 18; // list of int16_t + DT_INTS32 = 19; // list of int32_t + DT_INTS64 = 20; // list of int64_t + + DT_UINTS8 = 21; // list of uint8_t + DT_UINTS16 = 22; // list of uint16_t + DT_UINTS32 = 23; // list of uint32_t + DT_UINTS64 = 24; // list of uint64_t + + DT_FLOATS16 = 25; // list of float16 + DT_FLOATS32 = 26; // list of float32 + DT_FLOATS64 = 27; // list of float64 + + DT_STRINGS = 28; // list of string + DT_TENSORS = 29; // list of tensor + DT_GRAPHS = 30; // list of graph + + DT_TUPLE = 31; // tuple + DT_LIST = 32; // list + DT_DICT = 33; // dictionary + + // other types + DT_NONE = 34; // None + DT_SYM_INST = 35; // Symbolic Key Instance + + // type related type + DT_BASE_INT = 36; // type generic int + DT_BASE_UINT = 37; // type generate unsigned int + DT_BASE_FLOAT = 38; // type generate float + DT_TYPE = 39; // type type + DT_ANY = 40; // type any + DT_REFKEY = 41; // type refkey + DT_REF = 42; // type ref + DT_COMPLEX64 = 43; // list of complex64 + DT_COMPLEX128 = 44; // list of complex128 + DT_BASE_COMPLEX = 45; // type generate complex + + // bfloat type + DT_BFLOAT16 = 46; // bfloat16 + DT_BFLOATS16 = 47; // list of bfloat16 + + // quant type + DT_INT4 = 48; // int4 + + // slice type + DT_SLICE = 49; +} + +// Value definition for attribute value or parameter default value +message ValueProto { + // data type of value + optional DataType dtype = 1; // discriminator that indicates which field below is in use + + // Exactly ONE of the following fields must be present for this version of the IR + optional bool bool_val = 2; // bool + optional int64 int_val = 3; // int + optional uint64 uint_val = 4; // uint + optional float float_val = 5; // float + optional double double_val = 6; // double + optional string str_val = 7; // string + optional TensorProto tensor_val = 8; // tensor value + optional GraphProto graph = 9; // graph + + repeated bool bool_vals = 10; // list of bool + repeated int64 int_vals = 11; // list of int + repeated uint64 uint_vals = 12; // list of uint + repeated float float_vals = 13; // list of float + repeated double double_vals = 14; // list of double + repeated string str_vals = 15; // list of string + repeated TensorProto tensor_vals = 16; // list of tensor value + repeated GraphProto graphs = 17; // list of graph + + // tuple or list + repeated ValueProto values = 18; // tuple, list of value + + // dictionary + repeated NamedValueProto dict_val = 19; // dictionary info + + // filed for type type + optional TypeProto type_val = 20; // type type info +} + +message AttributeProto { + optional string name = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +message NamedValueProto { + optional string key = 1; // attribute name + optional ValueProto value = 2; // attribute value +} + +// Defines a tensor shape. +message TensorShapeProto { + // One dimension of the tensor. + message Dimension { + // Size of the tensor in that dimension. + // This value must be >= -1, but values of -1 are reserved for "unknown" + // shapes (values of -1 mean "unknown" dimension). + optional int64 size = 1; + + // Optional name of the tensor dimension. + optional string name = 2; + }; + + repeated Dimension dim = 1; +} + +// Types for graph input(parameter) and output +message TypeProto { + + message Tensor { + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType elem_type = 1; + optional TensorShapeProto shape = 2; // for scalar, this field is not set + } + + // tuple type + message Sequence { + // The type and optional shape of elements of the tuple. + repeated TypeProto elem_types = 1; + }; + + // data type + optional DataType data_type = 1; + + oneof value { + // The type of a tensor. + Tensor tensor_type = 2; + + // The type of a tuple. + Sequence sequence_type = 3; + } +} + +// Defines information on graph parameters, including the name, the type, and +// the default value of parameter if exists. +message ParameterProto { + optional string name = 1; // parameter name + optional TypeProto type = 2; // parameter type + optional ValueProto default_val = 3; // default value of parameter if exists +} + +// Defines graph output information +message OutputProto { + optional string name = 1; // output node name + optional TypeProto type = 2; // output node type +} + +// Define node input information +message InputProto { + enum EdgeType { + DATA_EDGE = 0; // data edge + CONTROL_EDGE = 1; // control edge + } + + optional string name = 1; + optional EdgeType type = 2; +} + +// Nodes +// +// Computation graphs are made up of a DAG of nodes, which represent what is +// commonly called a "layer" or "pipeline stage" in machine learning frameworks. +// +// For example, it can be a node of type "Conv" that takes in an image, a filter +// tensor and a bias tensor, and produces the convolved output. +message NodeProto { + repeated InputProto input = 1; // namespace Value + optional string name = 2; // namespace Value + + // The symbolic identifier of the Operator to execute. + optional string op_type = 3; // namespace Operator + // The domain of the OperatorSet that specifies the operator named by op_type. + optional string scope = 4; // namespace Domain + + // Additional named attributes. + repeated AttributeProto attribute = 5; + + // Optional type info of this node + optional TypeProto output_type = 6; + + // other fields for debug + optional uint64 output_i = 7; + + // The full_name_with_scope of CNode + optional string full_name = 8; + + // Note: Id 9 is reserved for the source_address field of the debugger, please see debug_graph.proto + + // As same as the IR file instance name field. + optional string instance_name = 10; +} + +// Models +// +// ModelProto is a top-level file/container format for bundling a ML model and +// associating its computation graph with metadata. +// +// The semantics of the model are described by the associated GraphProto. +message ModelProto { + // ir version + optional int64 ir_version = 1; + + // Domain name of the model. + // We use reverse domain names as name space indicators. For example: + // `com.facebook.fair` or `com.microsoft.cognitiveservices` + // + // Together with `model_version` and GraphProto.name, this forms the unique identity of + // the graph. + optional string domain = 2; + + // The version of the graph encoded. See Version enum below. + optional int64 model_version = 3; + + // The parameterized graph that is evaluated to execute the model. + optional GraphProto graph = 4; + + // metadata info of operators + optional OperatorSetProto metadata_operators = 5; +}; + +message OperatorProto { + optional string name = 1; // used as key, must be distinct + optional bytes config = 2; // operator config info + optional bytes obj_info = 3; // operator related object info, e.g. content of operator binary or name +}; + +message OperatorSetProto { + repeated OperatorProto operators = 1; + optional string summary = 2; // summary info of operators, e.g. file position of operators file +} + +// Graphs +// +// A graph defines the computational logic of a model and is comprised of a parameterized +// list of nodes that form a directed acyclic graph based on their inputs and outputs. +// This is the equivalent of the "network" or "graph" in many deep learning +// frameworks. +message GraphProto { + // The nodes in the graph, sorted topologically. + repeated NodeProto node = 1; + + // The name of the graph. + optional string name = 2; // namespace Graph + + // The parameters(inputs) and outputs of the graph. + repeated ParameterProto parameters = 3; + repeated OutputProto outputs = 4; + + // Constants used in this graph + repeated NamedValueProto const_vals = 5; +} + +// Tensors +// +// A serialized tensor value. +message TensorProto { + // The shape of the tensor. + repeated int64 dims = 1; + + // The data type of the tensor. + // This field MUST have a valid DataType value except DT_TENSOR + optional DataType data_type = 2; + + // Tensor content must be organized in row-major order. + // + // Depending on the data_type field, exactly one of the fields below with + // name ending in _data is used to store the elements of the tensor. + + // For float values + repeated float float_data = 3 [packed = true]; + + // For int32, uint8, int8, uint16, int16, and bool values + // When this field is present, the data_type field MUST be + // INT32, INT16, INT8, UINT16, UINT8, or BOOL + repeated int32 int32_data = 4 [packed = true]; + + // For int64. + // When this field is present, the data_type field MUST be INT64 + repeated int64 int64_data = 5 [packed = true]; + + // For double + // When this field is present, the data_type field MUST be DOUBLE + repeated double double_data = 6 [packed = true]; + + // For uint64 and uint32 values + // When this field is present, the data_type field MUST be + // UINT32 or UINT64 + repeated uint64 uint64_data = 7 [packed = true]; + + // Store raw tensor content. When this raw_data field is used to store tensor value, + // elements MUST be stored in as fixed-width, little-endian order. + optional bytes raw_data = 8; +} diff --git a/plugins/mindstudio-insight-plugins/proto/mindspore_summary.proto b/plugins/mindstudio-insight-plugins/proto/mindspore_summary.proto new file mode 100644 index 0000000000000000000000000000000000000000..b9c67279d2c06a12b86932ed4bf8edb3fbe1da22 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/mindspore_summary.proto @@ -0,0 +1,188 @@ +/** + * Copyright 2019-2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax = "proto2"; + +package mindspore.irpb; +option cc_enable_arenas = true; + +// The ANF IR define, include the tensor and graph define +import "mindspore_anf_ir.proto"; + +// Event Protocol buffer, Top define +message Event { + // Timestamp + required double wall_time = 1; + + // The step of train. + optional int64 step = 2; + + oneof what { + // An event file was started, with the specified version. + // Now version is "MindSpore.Event:1" + string version = 3; + + // GraphDef. + GraphProto graph_def = 4; + + // Summary data + Summary summary = 5; + + Explain explain = 6; + } +} + +message LossLandscape{ + message Point { + optional TensorProto x = 1; + optional TensorProto y = 2; + optional TensorProto z = 3; + } + + message LossPath { + repeated int32 intervals = 1; // step intervals or epoch intervals + optional Point points = 2; + } + + message Metadata { + optional string decomposition = 1; + optional string unit = 2; // step or epoch + optional int32 step_per_epoch = 3; + } + + optional Point landscape = 1; + optional LossPath loss_path = 2; + optional Metadata metadata = 3; // maybe only record by the first value + optional Point convergence_point = 4; + +} + +// A Summary is a set of named values that be produced regularly during training +message Summary { + message Image { + // Dimensions of the image. + required int32 height = 1; + required int32 width = 2; + // Valid colorspace values are: + // 1 - grayscale type + // 2 - grayscale + alpha type + // 3 - RGB type + // 4 - RGBA type + // 5 - DIGITAL_YUV type + // 6 - BGRA type + required int32 colorspace = 3; + // Image data in encoded format. Now only support the RGB. + required bytes encoded_image = 4; + } + + message Histogram { + message bucket{ + // Count number of values fallen in [left, left + width). + // For the right most bucket, range is [left, left + width]. + required double left = 1; + required double width = 2; + required int64 count = 3; + } + + repeated bucket buckets = 1; + optional int64 nan_count = 2; + optional int64 pos_inf_count = 3; + optional int64 neg_inf_count = 4; + + // max, min, sum will not take nan and inf into account. + // If there is no valid value in tensor, max will be nan, min will be nan, sum will be 0. + optional double max = 5; + optional double min = 6; + optional double sum = 7; + + // total number of values, including nan and inf + optional int64 count = 8; + } + + message Value { + // Tag name for the data. + required string tag = 1; + + // Value associated with the tag. + oneof value { + float scalar_value = 3; + Image image = 4; + TensorProto tensor = 8; + Histogram histogram = 9; + LossLandscape loss_landscape = 10; + } + } + + // Set of values for the summary. + repeated Value value = 1; +} + +message Explain { + message Inference{ + repeated float ground_truth_prob = 1; + repeated int32 predicted_label = 2; + repeated float predicted_prob = 3; + repeated float ground_truth_prob_sd = 4; + repeated float ground_truth_prob_itl95_low = 5; + repeated float ground_truth_prob_itl95_hi = 6; + repeated float predicted_prob_sd = 7; + repeated float predicted_prob_itl95_low = 8; + repeated float predicted_prob_itl95_hi = 9; + } + + message Explanation{ + optional string explain_method = 1; + optional int32 label = 2; + optional string heatmap_path = 3; + } + + message Benchmark{ + optional string benchmark_method = 1; + optional string explain_method = 2; + optional float total_score = 3; + repeated float label_score = 4; + } + + message Metadata{ + repeated string label = 1; + repeated string explain_method = 2; + repeated string benchmark_method = 3; + } + + message HocLayer { + optional float prob = 1; + repeated int32 box = 2; // List of repeated x, y, w, h + } + + message Hoc { + optional int32 label = 1; + optional string mask = 2; + repeated HocLayer layer = 3; + } + + optional int32 sample_id = 1; + optional string image_path = 2; // The Metadata and image path must have one fill in + repeated int32 ground_truth_label = 3; + + optional Inference inference = 4; + repeated Explanation explanation = 5; + repeated Benchmark benchmark = 6; + + optional Metadata metadata = 7; + optional string status = 8; // enum value: run, end + + repeated Hoc hoc = 9; // hierarchical occlusion counterfactual +} \ No newline at end of file diff --git a/plugins/mindstudio-insight-plugins/proto/resource_handle.proto b/plugins/mindstudio-insight-plugins/proto/resource_handle.proto new file mode 100644 index 0000000000000000000000000000000000000000..084a11cc6f44a6c7d2996ac1e943b132ed02bb8f --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/resource_handle.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; + +package tensorboard; + +import "tensor_shape.proto"; +import "types.proto"; + +option cc_enable_arenas = true; +option java_outer_classname = "ResourceHandle"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/resource_handle_go_proto"; + +// Protocol buffer representing a handle to a tensorflow resource. Handles are +// not valid across executions, but can be serialized back and forth from within +// a single run. +message ResourceHandleProto { + // Unique name for the device containing the resource. + string device = 1; + + // Container in which this resource is placed. + string container = 2; + + // Unique name of this resource. + string name = 3; + + // Hash code for the type of the resource. Is only valid in the same device + // and in the same execution. + uint64 hash_code = 4; + + // For debug-only, the name of the type pointed to by this handle, if + // available. + string maybe_type_name = 5; + + // Protocol buffer representing a pair of (data type, tensor shape). + message DtypeAndShape { + // Data type of the tensor. + DataType dtype = 1; + // Shape of the tensor. + TensorShapeProto shape = 2; + } + + // Data types and shapes for the underlying resource. + repeated DtypeAndShape dtypes_and_shapes = 6; + + reserved 7; +} diff --git a/plugins/mindstudio-insight-plugins/proto/summary.proto b/plugins/mindstudio-insight-plugins/proto/summary.proto new file mode 100644 index 0000000000000000000000000000000000000000..734d5e936384517a023d18bd3fa015e0bb5b67dd --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/summary.proto @@ -0,0 +1,132 @@ +syntax = "proto3"; + +package tensorboard; + +import public "histogram.proto"; +import "tensor.proto"; + +option cc_enable_arenas = true; +option java_outer_classname = "SummaryProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/summary_go_proto"; + +// Metadata associated with a series of Summary data +message SummaryDescription { + // Hint on how plugins should process the data in this series. + // Supported values include "scalar", "histogram", "image", "audio" + string type_hint = 1; +} + +// A SummaryMetadata encapsulates information on which plugins are able to make +// use of a certain summary value. +message SummaryMetadata { + message PluginData { + // The name of the plugin this data pertains to. + string plugin_name = 1; + + // The content to store for the plugin. The best practice is for this to be + // a binary serialized protocol buffer. + bytes content = 2; + } + + // Data that associates a summary with a certain plugin. + PluginData plugin_data = 1; + + // Display name for viewing in TensorBoard. + string display_name = 2; + + // Longform readable description of the summary sequence. Markdown supported. + string summary_description = 3; + + // Class of data stored in this time series. Required for compatibility with + // TensorBoard's generic data facilities (`DataProvider`, et al.). This value + // imposes constraints on the dtype and shape of the corresponding tensor + // values. See `DataClass` docs for details. + DataClass data_class = 4; +} + +enum DataClass { + // Unknown data class, used (implicitly) for legacy data. Will not be + // processed by data ingestion pipelines. + DATA_CLASS_UNKNOWN = 0; + // Scalar time series. Each `Value` for the corresponding tag must have + // `tensor` set to a rank-0 tensor of type `DT_FLOAT` (float32). + DATA_CLASS_SCALAR = 1; + // Tensor time series. Each `Value` for the corresponding tag must have + // `tensor` set. The tensor value is arbitrary, but should be small to + // accommodate direct storage in database backends: an upper bound of a few + // kilobytes is a reasonable rule of thumb. + DATA_CLASS_TENSOR = 2; + // Blob sequence time series. Each `Value` for the corresponding tag must + // have `tensor` set to a rank-1 tensor of bytestring dtype. + DATA_CLASS_BLOB_SEQUENCE = 3; +} + +// A Summary is a set of named values to be displayed by the +// visualizer. +// +// Summaries are produced regularly during training, as controlled by +// the "summary_interval_secs" attribute of the training operation. +// Summaries are also produced at the end of an evaluation. +message Summary { + message Image { + // Dimensions of the image. + int32 height = 1; + int32 width = 2; + // Valid colorspace values are + // 1 - grayscale + // 2 - grayscale + alpha + // 3 - RGB + // 4 - RGBA + // 5 - DIGITAL_YUV + // 6 - BGRA + int32 colorspace = 3; + // Image data in encoded format. All image formats supported by + // image_codec::CoderUtil can be stored here. + bytes encoded_image_string = 4; + } + + message Audio { + // Sample rate of the audio in Hz. + float sample_rate = 1; + // Number of channels of audio. + int64 num_channels = 2; + // Length of the audio in frames (samples per channel). + int64 length_frames = 3; + // Encoded audio data and its associated RFC 2045 content type (e.g. + // "audio/wav"). + bytes encoded_audio_string = 4; + string content_type = 5; + } + + message Value { + // This field is deprecated and will not be set. + string node_name = 7; + + // Tag name for the data. Used by TensorBoard plugins to organize data. Tags + // are often organized by scope (which contains slashes to convey + // hierarchy). For example: foo/bar/0 + string tag = 1; + + // Contains metadata on the summary value such as which plugins may use it. + // Take note that many summary values may lack a metadata field. This is + // because the FileWriter only keeps a metadata object on the first summary + // value with a certain tag for each tag. TensorBoard then remembers which + // tags are associated with which plugins. This saves space. + SummaryMetadata metadata = 9; + + // Value associated with the tag. + oneof value { + float simple_value = 2; + bytes obsolete_old_style_histogram = 3; + Image image = 4; + HistogramProto histo = 5; + Audio audio = 6; + TensorProto tensor = 8; + } + } + + // Set of values for the summary. + repeated Value value = 1; +} diff --git a/plugins/mindstudio-insight-plugins/proto/tensor.proto b/plugins/mindstudio-insight-plugins/proto/tensor.proto new file mode 100644 index 0000000000000000000000000000000000000000..251ac2b6b7fa85969c193b657080e0440ac7a700 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/tensor.proto @@ -0,0 +1,101 @@ +syntax = "proto3"; + +package tensorboard; + +import "resource_handle.proto"; +import "tensor_shape.proto"; +import "types.proto"; + +option cc_enable_arenas = true; +option java_outer_classname = "TensorProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_go_proto"; + +// Protocol buffer representing a tensor. +message TensorProto { + // Data type of the tensor. + DataType dtype = 1; + + // Shape of the tensor. TODO(touts): sort out the 0-rank issues. + TensorShapeProto tensor_shape = 2; + + // Only one of the representations below is set, one of "tensor_contents" and + // the "xxx_val" attributes. We are not using oneof because as oneofs cannot + // contain repeated fields it would require another extra set of messages. + + // Version number. + // + // In version 0, if the "repeated xxx" representations contain only one + // element, that element is repeated to fill the shape. This makes it easy + // to represent a constant Tensor with a single value. + int32 version_number = 3; + + // Serialized raw tensor content from either Tensor::AsProtoTensorContent or + // memcpy in tensorflow::grpc::EncodeTensorToByteBuffer. This representation + // can be used for all tensor types. The purpose of this representation is to + // reduce serialization overhead during RPC call by avoiding serialization of + // many repeated small items. + bytes tensor_content = 4; + + // Type specific representations that make it easy to create tensor protos in + // all languages. Only the representation corresponding to "dtype" can + // be set. The values hold the flattened representation of the tensor in + // row major order. + + // DT_HALF, DT_BFLOAT16. Note that since protobuf has no int16 type, we'll + // have some pointless zero padding for each value here. + repeated int32 half_val = 13 [packed = true]; + + // DT_FLOAT. + repeated float float_val = 5 [packed = true]; + + // DT_DOUBLE. + repeated double double_val = 6 [packed = true]; + + // DT_INT32, DT_INT16, DT_UINT16, DT_INT8, DT_UINT8. + repeated int32 int_val = 7 [packed = true]; + + // DT_STRING + repeated bytes string_val = 8; + + // DT_COMPLEX64. scomplex_val(2*i) and scomplex_val(2*i+1) are real + // and imaginary parts of i-th single precision complex. + repeated float scomplex_val = 9 [packed = true]; + + // DT_INT64 + repeated int64 int64_val = 10 [packed = true]; + + // DT_BOOL + repeated bool bool_val = 11 [packed = true]; + + // DT_COMPLEX128. dcomplex_val(2*i) and dcomplex_val(2*i+1) are real + // and imaginary parts of i-th double precision complex. + repeated double dcomplex_val = 12 [packed = true]; + + // DT_RESOURCE + repeated ResourceHandleProto resource_handle_val = 14; + + // DT_VARIANT + repeated VariantTensorDataProto variant_val = 15; + + // DT_UINT32 + repeated uint32 uint32_val = 16 [packed = true]; + + // DT_UINT64 + repeated uint64 uint64_val = 17 [packed = true]; + + // DT_FLOAT8_*, use variable-sized set of bytes + // (i.e. the equivalent of repeated uint8, if such a thing existed). + bytes float8_val = 18; +} + +// Protocol buffer representing the serialization format of DT_VARIANT tensors. +message VariantTensorDataProto { + // Name of the type of objects being serialized. + string type_name = 1; + // Portions of the object that are not Tensors. + bytes metadata = 2; + // Tensors contained within objects being serialized. + repeated TensorProto tensors = 3; +} diff --git a/plugins/mindstudio-insight-plugins/proto/tensor_shape.proto b/plugins/mindstudio-insight-plugins/proto/tensor_shape.proto new file mode 100644 index 0000000000000000000000000000000000000000..9da8a082b5129973676357c5ff7f82b11bd65c94 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/tensor_shape.proto @@ -0,0 +1,47 @@ +// Protocol buffer representing the shape of tensors. + +syntax = "proto3"; + +option cc_enable_arenas = true; +option java_outer_classname = "TensorShapeProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_shape_go_proto"; + +package tensorboard; + +// Dimensions of a tensor. +message TensorShapeProto { + // One dimension of the tensor. + message Dim { + // Size of the tensor in that dimension. + // This value must be >= -1, but values of -1 are reserved for "unknown" + // shapes (values of -1 mean "unknown" dimension). Certain wrappers + // that work with TensorShapeProto may fail at runtime when deserializing + // a TensorShapeProto containing a dim value of -1. + int64 size = 1; + + // Optional name of the tensor dimension. + string name = 2; + }; + + // Dimensions of the tensor, such as {"input", 30}, {"output", 40} + // for a 30 x 40 2D tensor. If an entry has size -1, this + // corresponds to a dimension of unknown size. The names are + // optional. + // + // The order of entries in "dim" matters: It indicates the layout of the + // values in the tensor in-memory representation. + // + // The first entry in "dim" is the outermost dimension used to layout the + // values, the last entry is the innermost dimension. This matches the + // in-memory layout of RowMajor Eigen tensors. + // + // If "dim.size()" > 0, "unknown_rank" must be false. + repeated Dim dim = 2; + + // If true, the number of dimensions in the shape is unknown. + // + // If true, "dim.size()" must be 0. + bool unknown_rank = 3; +}; diff --git a/plugins/mindstudio-insight-plugins/proto/types.proto b/plugins/mindstudio-insight-plugins/proto/types.proto new file mode 100644 index 0000000000000000000000000000000000000000..89d5411419ad7fda1375f6588208f34b07f4661f --- /dev/null +++ b/plugins/mindstudio-insight-plugins/proto/types.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; + +package tensorboard; + +option cc_enable_arenas = true; +option java_outer_classname = "TypesProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/types_go_proto"; + +// (== suppress_warning documentation-presence ==) +// DISABLED.IfChange +enum DataType { + // Not a legal value for DataType. Used to indicate a DataType field + // has not been set. + DT_INVALID = 0; + + // Data types that all computation devices are expected to be + // capable to support. + DT_FLOAT = 1; + DT_DOUBLE = 2; + DT_INT32 = 3; + DT_UINT8 = 4; + DT_INT16 = 5; + DT_INT8 = 6; + DT_STRING = 7; + DT_COMPLEX64 = 8; // Single-precision complex + DT_INT64 = 9; + DT_BOOL = 10; + DT_QINT8 = 11; // Quantized int8 + DT_QUINT8 = 12; // Quantized uint8 + DT_QINT32 = 13; // Quantized int32 + DT_BFLOAT16 = 14; // Float32 truncated to 16 bits. + DT_QINT16 = 15; // Quantized int16 + DT_QUINT16 = 16; // Quantized uint16 + DT_UINT16 = 17; + DT_COMPLEX128 = 18; // Double-precision complex + DT_HALF = 19; + DT_RESOURCE = 20; + DT_VARIANT = 21; // Arbitrary C++ data types + DT_UINT32 = 22; + DT_UINT64 = 23; + DT_FLOAT8_E5M2 = 24; // 5 exponent bits, 2 mantissa bits. + DT_FLOAT8_E4M3FN = 25; // 4 exponent bits, 3 mantissa bits, finite-only, with + // 2 NaNs (0bS1111111). + // TODO - b/299182407: Leaving room for remaining float8 types. + // DT_FLOAT8_E4M3FNUZ = 26; + // DT_FLOAT8_E4M3B11FNUZ = 27; + // DT_FLOAT8_E5M2FNUZ = 28; + DT_INT4 = 29; + DT_UINT4 = 30; + + // Do not use! These are only for TF1's obsolete reference Variables. + // Every enum above should have a corresponding value below (verified by + // types_test). + DT_FLOAT_REF = 101; + DT_DOUBLE_REF = 102; + DT_INT32_REF = 103; + DT_UINT8_REF = 104; + DT_INT16_REF = 105; + DT_INT8_REF = 106; + DT_STRING_REF = 107; + DT_COMPLEX64_REF = 108; + DT_INT64_REF = 109; + DT_BOOL_REF = 110; + DT_QINT8_REF = 111; + DT_QUINT8_REF = 112; + DT_QINT32_REF = 113; + DT_BFLOAT16_REF = 114; + DT_QINT16_REF = 115; + DT_QUINT16_REF = 116; + DT_UINT16_REF = 117; + DT_COMPLEX128_REF = 118; + DT_HALF_REF = 119; + DT_RESOURCE_REF = 120; + DT_VARIANT_REF = 121; + DT_UINT32_REF = 122; + DT_UINT64_REF = 123; + DT_FLOAT8_E5M2_REF = 124; + DT_FLOAT8_E4M3FN_REF = 125; + // TODO - b/299182407: Leaving room for remaining float8 types. + // DT_FLOAT8_E4M3FNUZ_REF = 126; + // DT_FLOAT8_E4M3B11FNUZ_REF = 127; + // DT_FLOAT8_E5M2FNUZ_REF = 128; + DT_INT4_REF = 129; + DT_UINT4_REF = 130; +} +// DISABLED.ThenChange( +// https://www.tensorflow.org/code/tensorflow/c/tf_datatype.h, +// https://www.tensorflow.org/code/tensorflow/go/tensor.go, +// https://www.tensorflow.org/code/tensorboard/compat/proto/tensor.cc, +// https://www.tensorflow.org/code/tensorboard/compat/proto/types.h, +// https://www.tensorflow.org/code/tensorboard/compat/proto/types.cc, +// https://www.tensorflow.org/code/tensorboard/compat/proto/dtypes.py, +// https://www.tensorflow.org/code/tensorboard/compat/proto/function.py) + +// Represents a serialized tf.dtypes.Dtype +message SerializedDType { + DataType datatype = 1; +} diff --git a/plugins/mindstudio-insight-plugins/tools/httpServer/CMakeLists.txt b/plugins/mindstudio-insight-plugins/tools/httpServer/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..5cef551d15ccfee1af0a69fa5283c5826e7a6c18 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/tools/httpServer/CMakeLists.txt @@ -0,0 +1,26 @@ +project(HttpServer) +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_C_STANDARD 11) + +set(EXECUTABLE_OUTPUT_PATH ${PROJECT_ROOT_DIR}/output) +aux_source_directory(. MAIN_SRC) +list(APPEND MAIN_SRC + ${MAIN_SRC} + ${U_SOCKETS_SRC}) +add_executable(${PROJECT_NAME} ${MAIN_SRC} + ../../plugin_core/src/Logger.cpp) +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + target_compile_options(${PROJECT_NAME} PUBLIC -g -O0) +endif () +target_compile_definitions(${PROJECT_NAME} PRIVATE PLUGINS_DIR="${PROJECT_ROOT_DIR}/output/plugins") +target_include_directories(${PROJECT_NAME} PUBLIC ${MAIN_INCLUDE}) +target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_ROOT_DIR}/plugin_core/include) +find_library(MSINSIGHT_LIB + NAMES msinsight + PATHS ${PROJECT_ROOT_PATH}/output/lib + NO_DEFAULT_PATH) +target_link_libraries(${PROJECT_NAME} PUBLIC ${MSINSIGHT_LIB}) +target_link_libraries(${PROJECT_NAME} PUBLIC uv_a) +if (CMAKE_SYSTEM_NAME MATCHES "Linux") + target_link_libraries(${PROJECT_NAME} PUBLIC stdc++fs) +endif () diff --git a/plugins/mindstudio-insight-plugins/tools/httpServer/HttpServer.cpp b/plugins/mindstudio-insight-plugins/tools/httpServer/HttpServer.cpp new file mode 100644 index 0000000000000000000000000000000000000000..9da0f552cacdda4a64aa0777f7cf9bbab20861e0 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/tools/httpServer/HttpServer.cpp @@ -0,0 +1,104 @@ +/* + * Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. + */ +#include +#include +#include "stringbuffer.h" +#include "writer.h" +#include "HttpServer.h" +#include "PluginsManager.h" +#include "Logger.h" + +using namespace Insight; +namespace Insight::Http { +using json_t = rapidjson::Value; +using document_t = rapidjson::Document; + +HttpServer &HttpServer::Instance() { + static HttpServer instance; + return instance; +} + +static inline std::string DumpJsonToStr(json_t &jsonSrc) { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + jsonSrc.Accept(writer); + return {buffer.GetString(), buffer.GetSize()}; +} + +bool HttpServer::Start() { + wsApp = std::make_unique(); + AddApiHandlers(); + wsApp->options("/*", [](auto *res, auto *req) { + res->end(); + }); + wsApp->listen("0.0.0.0", port, [](auto *token) { + if (token) { + LOG(LogRank::Info) << "http server start"; + } + }).run(); + return false; +} + +void HttpServer::AddApiHandlers() { + wsApp->get("test", [this](HttpResponse *res, auto *req) { + std::cout << "test" << std::endl; + res->tryEnd("", 0); + }); + auto &manager = Dic::Core::PluginsManager::Instance(); + Dic::Core::PluginsManager::LoadPlugins(); + for (const auto &[name, plugin]: manager.GetAllPlugins()) { + auto handlers = plugin->GetAllHandlers(); + for (const auto &[key, handler]: handlers) { + std::cout << "Add Handler:" << key << std::endl; + if (handler->GetApiType() == Dic::Core::API_TYPE::GET) { + std::cout << "Add Handler2" << std::endl; + AddGetHandler(std::string("/" + name + "/" + key), handler); + } else { + AddPostHandler(std::string("/" + name + "/" + key), handler); + } + } + } +} + +void HttpServer::AddGetHandler(std::string_view key, const std::shared_ptr handler) { + wsApp->get(key.data(), [handler, this](HttpResponse *res, HttpRequest *req) { + res->writeHeader("Access-Control-Allow-Origin", "*"); // allow CROS request + res->writeHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + res->writeHeader("Access-Control-Allow-Headers", "Content-Type"); + res->writeHeader("Access-Control-Allow-Credentials", "true"); + std::string result = GetBasicResult(); + if (handler->run(req->getQuery(), result)) { + } + res->tryEnd(result, result.size()); + }); +} + +void HttpServer::AddPostHandler(std::string_view key, const std::shared_ptr handler) { + wsApp->post(key.data(), [handler, this](HttpResponse *res, auto *req) { + res->onAborted([]() { + Loop::get()->defer([]() { + }); + }); + res->onData([res, handler, bodyBuffer = std::string(), this](std::string_view data, bool isEnd) mutable { + bodyBuffer.append(data); + if (isEnd) { + // 处理数据 + res->writeHeader("Access-Control-Allow-Origin", "*"); // allow CROS request + res->writeHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + res->writeHeader("Access-Control-Allow-Headers", "Content-Type"); + res->writeHeader("Access-Control-Allow-Credentials", "true"); + std::string result = GetBasicResult(); + bool sucess = handler->run(bodyBuffer, result); + // long data length can't send rightly with try end + res->end(result, true); + LOG(LogRank::Info) << "Response size:" << result.size(); + } + }); + }); +} + +std::string HttpServer::GetBasicResult() { + return R"({"body":{}, "msg":"", "errCode":0, "result":true})"; +} +} diff --git a/plugins/mindstudio-insight-plugins/tools/httpServer/HttpServer.h b/plugins/mindstudio-insight-plugins/tools/httpServer/HttpServer.h new file mode 100644 index 0000000000000000000000000000000000000000..52f468ade87afba8d5d0564c6c8a2be530a5d7d0 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/tools/httpServer/HttpServer.h @@ -0,0 +1,40 @@ +/* + * Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. + */ +#ifndef BOARD_HTTPSERVER_H +#define BOARD_HTTPSERVER_H + +#include +#include +#include "App.h" +#include "ApiHandler.h" + +namespace Insight::Http { +using namespace uWS; +using namespace Dic::Core; + +class HttpServer { +public: + static HttpServer &Instance(); + + bool Start(); + +private: + HttpServer() = default; + + ~HttpServer() = default; + + void AddApiHandlers(); + + void AddGetHandler(std::string_view key, const std::shared_ptr handler); + + void AddPostHandler(std::string_view key, const std::shared_ptr handler); + + static std::string GetBasicResult(); + + std::unique_ptr wsApp; + uint16_t port{6065}; +}; +} + +#endif // BOARD_HTTPSERVER_H diff --git a/plugins/mindstudio-insight-plugins/tools/httpServer/main.cpp b/plugins/mindstudio-insight-plugins/tools/httpServer/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..354b7384000ab5c053c3928794dd9641c09e1304 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/tools/httpServer/main.cpp @@ -0,0 +1,11 @@ +/* + * Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. + */ + +#include "HttpServer.h" + +using namespace Dic::Core; + +int main(int argc, char *argv[]) { + Insight::Http::HttpServer::Instance().Start(); +} \ No newline at end of file