diff --git a/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/HistoConceptDefs.h b/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/HistoConceptDefs.h new file mode 100644 index 0000000000000000000000000000000000000000..8fbaa4285e39712a3081668e5d94b6131b4b4b6a --- /dev/null +++ b/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/HistoConceptDefs.h @@ -0,0 +1,139 @@ +/* +* Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. + */ +#ifndef CONCEPTDEFS_H +#define CONCEPTDEFS_H +#include +#include + +#include "rapidjson.h" +#include "document.h" +#ifdef _WIN32 +#include +namespace fs = std::filesystem; +#else +#include +namespace fs = std::experimental::filesystem; +#endif +#include + +#include "proto/event.pb.h" +#include "proto/mindspore_summary.pb.h" + +namespace Insight::Histogram { + using value_t = rapidjson::Value; + using document_t = rapidjson::Document; + + enum class ParseDataType { + MindSpore_Summary = 0, + TF_EVENT = 1, + Unknown = 2 + }; + + enum ErrCode : int { + OK = 0, + INVALID_REQUEST_JSON, + REQUEST_INVALID_PARAM, + INVALID_PATH + }; + + // 直方图上每一条线,y轴的高度和属于哪一个step,由HistogramProto转化而来 + struct HistogramLine { + int64_t step_{0}; + std::vector bucket_; + std::vector bucketLimit_; + HistogramLine() = default; + HistogramLine(int64_t step, tensorboard::HistogramProto hisProto) { + step_ = step; + std::copy(hisProto.bucket().begin(), hisProto.bucket().end(), std::back_inserter(bucket_)); + std::copy(hisProto.bucket_limit().begin(), hisProto.bucket_limit().end(), + std::back_inserter(bucketLimit_)); + } + + HistogramLine(int64_t step, mindspore::irpb::Summary_Histogram histogram) { + step_ = step; + for (auto& bucket : histogram.buckets()) { + bucketLimit_.push_back(bucket.left() + bucket.width()); + bucket_.push_back(bucket.count()); + } + } + + value_t CreatHistoLineValue(document_t::AllocatorType& allocator) { + value_t histoGramLine(rapidjson::kObjectType); + // bucket_ 数据 + value_t bucket(rapidjson::kArrayType); + for (const auto& count: bucket_) { + bucket.PushBack(count, allocator); + } + value_t bucketLimit(rapidjson::kArrayType); + for (const auto& limit: bucketLimit_) { + bucketLimit.PushBack(limit, allocator); + } + histoGramLine.AddMember("step", step_, allocator); + histoGramLine.AddMember("bucket", bucket, allocator); + histoGramLine.AddMember("bucketLimit", bucketLimit, allocator); + return histoGramLine; + } + }; + + struct HistogramGraph { + std::vector hsitogramLines_; + // 记录上一次getdata请求的时候hsitogramLines_长度,初始未请求时是0 + // 下采样的缓存数据 + std::vector viewLines; + int reservoirNum = 50; + HistogramGraph() = default; + explicit HistogramGraph(const std::vector& hsitogramLines) : hsitogramLines_(hsitogramLines) {} + void AddValue(const HistogramLine& line) { + hsitogramLines_.push_back(line); + } + void MergeData(const HistogramGraph& graph) { + hsitogramLines_.insert(hsitogramLines_.end(),graph.hsitogramLines_.begin(), graph.hsitogramLines_.end()); + } + + // 对数据进行下采样 + void SetdDownsampleCatchLines() { + // 清空上一次的下采样数据重新进行下采样 + viewLines.clear(); + std::vector indices(hsitogramLines_.size() - 1); + std::iota(indices.begin(), indices.end(), 0); + + // 使用固定的随机数种子0进行随机打乱 + std::default_random_engine generator(0); + std::shuffle(indices.begin(), indices.end(), generator); + + // 选择前k - 1 个元素,并添加最后一个元素 + indices.resize(reservoirNum - 1); + std::sort(indices.begin(), indices.end()); + indices.push_back(hsitogramLines_.size() - 1); + + std::vector result; + for (int idx : indices) { + viewLines.push_back(hsitogramLines_[idx]); + } + } + + void UpdateViewData() { + // hsitogramLines_ 小于 50, 不抽样返回原始数据 + if (hsitogramLines_.size() < reservoirNum) { + viewLines = hsitogramLines_; + return; + } + // 更新缓存的下采样数据 + SetdDownsampleCatchLines(); + } + + value_t CreatHistogramValue(document_t::AllocatorType& allocator) { + // 获取json说明接收到了getdata的请求,所以更新下采样数据 + UpdateViewData(); + // 转成json格式 + value_t hsitogramLinesValue(rapidjson::kArrayType); + for (auto& hsitogramLine: viewLines) { + value_t histogram = hsitogramLine.CreatHistoLineValue(allocator); + hsitogramLinesValue.PushBack(histogram, allocator); + } + return hsitogramLinesValue; + } + }; +} +#endif //CONCEPTDEFS_H diff --git a/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/RequestDef.h b/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/RequestDef.h new file mode 100644 index 0000000000000000000000000000000000000000..55a63a98021ddce904a247b27c4d9cabe83f62b8 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/RequestDef.h @@ -0,0 +1,99 @@ +/* +* Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. + */ +#ifndef REQUEST_DEF_H +#define REQUEST_DEF_H + +#include +#include +#include "rapidjson.h" +#include "document.h" + +#include "HistoConceptDefs.h" + +namespace Insight::Histogram { + using document_t = rapidjson::Document; + using value_t = rapidjson::Value; + using size_type_t =rapidjson::SizeType; + struct ImportRequest { + std::vector rootPathList; + ImportRequest() = default; + explicit ImportRequest(std::string_view request) { + std::string jsonString(request); + document_t document; + if (document.Parse(jsonString.c_str()).HasParseError()) { + return; + } + if (document.HasMember("rootPaths") && document["rootPaths"].IsArray()) { + const value_t& pathList = document["rootPaths"]; + for (size_type_t i = 0; i < pathList.Size(); i++) { + rootPathList.emplace_back(pathList[i].GetString()); + } + } + } + }; + + // 获取新增文件的接口,返回rootpath下面路径列表,没有入参 filrlist + struct GetNewFilePathRequest { + }; + + // 更新file和tag的映射关系, 返回的是file tagsd对应关系,这部分一定是没有解析过的文件最新导入进来 + struct AddImportFileRequest { + std::vector filepathList; //确定的具体文件 + AddImportFileRequest() = default; + explicit AddImportFileRequest(std::string_view request) { + std::string jsonString(request); + document_t document; + if (document.Parse(jsonString.c_str()).HasParseError()) { + return; + } + if (document.HasMember("filePathList") && document["filePathList"].IsArray()) { + const value_t& pathList = document["filePathList"]; + for (size_type_t i = 0; i < pathList.Size(); i++) { + filepathList.emplace_back(pathList[i].GetString()); + } + } + } + }; + + /* + * 前端发送的json请求格式如下 + * { + * "filePathTotags": [ + * {"filepath": "/home/text/path1", "tag": "tagXXX1"}, + * {"filepath": "/home/text/path2", "tag": "tagXXX2"}, + * ] + * } + **/ + // 获取具体某个图的时候,会限定是哪个文件的哪个tag的图。一个文件可能有多个 + struct GetHistoDataRequest { + std::map> filePathTotags; + GetHistoDataRequest() = default; + explicit GetHistoDataRequest(std::string_view request) + { + std::string jsonString(request); + document_t document; + if (document.Parse(jsonString.c_str()).HasParseError()) { + return; + } + if (document.HasMember("filePathToTags") && document["filePathToTags"].IsArray()) { + const value_t& pathList = document["filePathToTags"]; + for (size_type_t i = 0; i < pathList.Size(); i++) { + const value_t& pathToTag = pathList[i]; + if (!pathToTag.HasMember("filePath") || !pathToTag.HasMember("tag")) { + continue; + } + if (!pathToTag["filePath"].IsString() || !pathToTag["tag"].IsString()) { + continue; + } + std::string path = pathToTag["filePath"].GetString(); + if (filePathTotags.find(path) == filePathTotags.end()) { + filePathTotags[path] = std::set(); + } + filePathTotags[path].insert(pathToTag["tag"].GetString()); + } + } + } + }; +} +#endif //REQUEST_DEF_H diff --git a/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/ResponseDef.h b/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/ResponseDef.h new file mode 100644 index 0000000000000000000000000000000000000000..1fb9e7874a7a638f860b8537ca3e22f248088fc4 --- /dev/null +++ b/plugins/mindstudio-insight-plugins/Histogram/server/src/defs/ResponseDef.h @@ -0,0 +1,162 @@ +/* +* Copyright (c), Huawei Technologies Co., Ltd. 2024-2024.All rights reserved. +*/ + +#ifndef RESPONSE_DEF_H +#define RESPONSE_DEF_H + +#include +#include +#include +#include +#include "rapidjson.h" +#include "document.h" + +#include "defs/HistoConceptDefs.h" + +namespace Insight::Histogram { + +using allocator_t = rapidjson::Document::AllocatorType; +using document_t = rapidjson::Document; +using value_t = rapidjson::Value; +using size_type_t =rapidjson::SizeType; + +struct ResponseDef { + std::string message_; + bool result_{true}; + int errCode_{0}; + ResponseDef() = default; + ResponseDef(const std::string & message, bool result, int errCode) : + message_(message), result_(result), errCode_(errCode) {} + + virtual value_t CreatDataValue(document_t::AllocatorType& allocator) { + value_t body(rapidjson::kObjectType); + return body; + } + + document_t CreatBasicJson() { + document_t document; + document.SetObject(); + document_t::AllocatorType& allocator = document.GetAllocator(); + document.AddMember("errCode", errCode_, allocator); + document.AddMember("msg", value_t().SetString(message_.c_str(), allocator), allocator); + document.AddMember("result", result_, allocator); + + value_t body = CreatDataValue(allocator); + document.AddMember("body", body, allocator); + return document; + } + + std::string ToJsonString() + { + document_t document = CreatBasicJson(); + // json 转换成string格式 + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + document.Accept(writer); + + std::string jsonString = buffer.GetString(); + return jsonString; + } + virtual ~ResponseDef() = default; +}; + +struct ImportResponse : public ResponseDef { + ImportResponse() = default; + explicit ImportResponse(const std::map> &filePathToTags) : filePathToTags_(filePathToTags) {} + ImportResponse(const std::string & message, bool result, int errCode, const std::map> &filePathToTags) : + ResponseDef(message, result, errCode), filePathToTags_(filePathToTags) {} + std::map> filePathToTags_; + // 数据转换成json格式,返回给前端 + value_t CreatDataValue(document_t::AllocatorType& allocator) override + { + value_t data(rapidjson::kArrayType); + for (const auto& [filePath, tags] : filePathToTags_) { + // filepath是一个json对象 + value_t entry(rapidjson::kObjectType); + entry.AddMember("filePath", value_t().SetString(filePath.c_str(), allocator), allocator); + // tags是一个集合,新建一个数组对象 + value_t tagList(rapidjson::kArrayType); + for (const auto& tag : tags) { + tagList.PushBack(value_t().SetString(tag.c_str(), allocator), allocator); + } + entry.AddMember("tagList", tagList, allocator); + data.PushBack(entry, allocator); + } + value_t body(rapidjson::kObjectType); + body.AddMember("data", data, allocator); + return body; + } +}; + +// 获取采集路径下的新增文件,request的时候不需要参数 +struct GetNewFilesResponse : public ResponseDef { + std::vector filePaths_; + GetNewFilesResponse() = default; + explicit GetNewFilesResponse(const std::vector &filePaths) : filePaths_(filePaths) {} + GetNewFilesResponse(const std::string & message, bool result, int errCode, + const std::vector &filePaths) : ResponseDef(message, result, errCode), filePaths_(filePaths) {} + // 数据转换成json格式,返回给前端 + value_t CreatDataValue(document_t::AllocatorType& allocator) override + { + value_t data(rapidjson::kArrayType); + for (const auto& filePath: filePaths_) { + data.PushBack(value_t().SetString(filePath.c_str(), allocator), allocator); + } + value_t body(rapidjson::kObjectType); + body.AddMember("data", data, allocator); + return body; + } +}; + +// 返回数据按照文件 - tag - 直方图的形式 +struct GetHistoDataResponse : public ResponseDef{ + std::map> filePathToInfo_; + std::map> filePathToNewTags_; + GetHistoDataResponse() = default; + explicit GetHistoDataResponse(const std::map> &filePathToInfo, + const std::map> &filePathToNewTags) : + filePathToInfo_(filePathToInfo), filePathToNewTags_(filePathToNewTags) {} + GetHistoDataResponse(const std::string & message, bool result, int errCode, + const std::map> &filePathToInfo, + const std::map> &filePathToNewTags) : + ResponseDef(message, result, errCode), filePathToInfo_(filePathToInfo), + filePathToNewTags_(filePathToNewTags) {} + // 数据转换成json格式,返回给前端 + value_t CreatDataValue(document_t::AllocatorType& allocator) override + { + value_t data(rapidjson::kArrayType); + for (const auto& filePair: filePathToInfo_) { + for (const auto& tagPair: filePair.second) { + value_t filetagHisto(rapidjson::kObjectType); + filetagHisto.AddMember("filePath", value_t().SetString(filePair.first.c_str(), allocator), allocator); + filetagHisto.AddMember("tag", value_t().SetString(tagPair.first.c_str(), allocator), allocator); + HistogramGraph histogram = tagPair.second; + filetagHisto.AddMember("histogramGraph", histogram.CreatHistogramValue(allocator), allocator); + + data.PushBack(filetagHisto, allocator); + } + } + + value_t newTags(rapidjson::kArrayType); + for (const auto& [filePath, tags] : filePathToNewTags_) { + // filepath是一个json对象 + value_t entry(rapidjson::kObjectType); + entry.AddMember("filePath", value_t().SetString(filePath.c_str(), allocator), allocator); + // tags是一个集合,新建一个数组对象 + value_t tagList(rapidjson::kArrayType); + for (const auto& tag : tags) { + tagList.PushBack(value_t().SetString(tag.c_str(), allocator), allocator); + } + entry.AddMember("tagList", tagList, allocator); + newTags.PushBack(entry, allocator); + } + value_t body(rapidjson::kObjectType); + body.AddMember("data", data, allocator); + body.AddMember("newTags", newTags, allocator); + return body; + } +}; + +} +#endif //RESPONSE_DEF_H