diff --git a/README.md b/README.md index c4a4f7973b36754e8642e71792ad1d64d454bafb..b3ce2d0814ab0668af114e5b4e53fdb2d4a6088f 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,178 @@ -# logging +# AISBench打点组件logging使用说明 +## 工具介绍 +### 名词定义 +|名词|定义| +| --- | --- | +|点事件|模型推理/训练过程中瞬间发生的事件、或是对某些值的记录| +|持续事件|模型推理/训练过程中会持续一段时间的事件| +|stubs程序|执行标准AI测试时，在被测试者端运行的用于拉起模型负载的控制程序| +### 工具概述 +AISBench打点组件logging（后简称logging工具）提供标准化接口，用于在模型负载中进行打点，记录模型运行数据。并且提供汇总上报功能，将多卡数据汇总上报stubs程序。logging工具可以和stubs搭配使用，也可以单独用于脚本中的打点。 -#### 介绍 -{**以下是 Gitee 平台说明，您可以替换此简介** -Gitee 是 OSCHINA 推出的基于 Git 的代码托管平台（同时支持 SVN）。专为开发者提供稳定、高效、安全的云端软件开发协作平台 -无论是个人、团队、或是企业，都能够用 Gitee 实现代码托管、项目管理、协作开发。企业项目请看 [https://gitee.com/enterprises](https://gitee.com/enterprises)} +### 工具依赖 +``` +os: linux +python version: >3.6 +``` -#### 软件架构 -软件架构说明 +## 安装和卸载 +### 安装logging -#### 安装教程 +1. 解压Stubs程序包`Ais-Benchmark-Stubs--.tar.gz`并进入子目录。（arch表示CPU架构，version表示Stubs版本号） +2. 进入Stubs程序包中的`dependencies/logging/`目录，确认路径下有文件`ais_bench_logging--py3-none-linux_.whl`, 用pip安装： + ``` + pip install ais_bench_logging--py3-none-linux_.whl --force-reinstall + ``` +3. 输入`pip show ais_bench_logging`确认安装完成，命令行打屏如下： + ``` + Name: ais-bench-logging + Version: 2.1 + Summary: ais_bench logging + Home-page: + Author: + Author-email: + License: + Location: /xxxx/site-packages + Requires: + Required-by: + ``` +### 卸载logging +如果要卸载打点组件logging，可以使用如下命令： +```bash +pip uninstall ais_bench_logging +``` -1. xxxx -2. xxxx -3. xxxx +## logging python API使用方法 +### 调用示例 +确保ais_bench logging工具已安装，调用API方法如下： +```python +from ais_bench.logging import +``` +其中`some function`表示对应的接口函数。 +### API使用介绍 +#### init（初始化接口） + 函数原型 -#### 使用说明 + ```python + init(task_type: str, log_dir:str) + ``` +需要传入 +- 本次记录的模型运行任务的类型，可接受"training"或者"inference"。 +- 落盘文件的目录。 -1. xxxx -2. xxxx -3. xxxx +用于初始化。初始化失败，则在后续的打点中只会执行打屏，不会执行落盘。 -#### 参与贡献 +#### start（持续事件开始打点接口） +函数原型 -1. Fork 本仓库 -2. 新建 Feat_xxx 分支 -3. 提交代码 -4. 新建 Pull Request + ```python + start(event_name: str, sample_num: int) + ``` +需要传入: +- 事件名称。 +- 样本数量。 +用于记录持续事件的开始，传入的事件名称如果不是规定字段，则只会进行打屏，不会对此打点做记录。 -#### 特技 -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目 -5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) +#### end（持续事件结束打点接口） +函数原型 + + ```python + end(event_name: str, sample_num: int) + ``` +需要传入： +- 事件名称。 +- 样本数量。 + +用于记录持续事件的结束，传入的事件名称如果不是规定字段，则只会进行打屏，不会对此打点做记录。 + +#### event（点事件打点接口） +函数原型 + + ```python + event(event_name: str, value: str) + ``` +需要传入： +- 事件名称。 +- 事件对应的值。 + +用于记录持续事件的结束，传入的事件名称如果不是规定字段，则只会进行打屏，不会对此打点做记录。 + +#### finish（后处理接口） +函数原型 + + ```python + finish() + ``` +和init配套使用，在所有打点完成之后调用，实现数据的落盘。 + +#### collect_report（汇总上报接口） +函数原型 + + ```python + collect_report(task_type: str, log_dirs: List[str]) + ``` +需要传入： +- 任务类型，可接受"training"或者"inference"。 +- 数据文件目录的列表。 + +用于从指定目录中获取指定任务类型的性能数据文件，并将其上报stubs程序。多次使用需要注意清理历史json文件。 + + +### 附录 + +#### 调试方法 +设置日志等级，以获取调试信息。1表示debug，2表示info，3表示warning，4表示error +```shell +export AISBENCH_LOG_LEVEL=1 +``` + +#### 整体打点规则 +当logging模块和stubs配合使用时，对于训练和推理都需要记录"result"点事件。 + +#### 训练任务持续事件规定字段 + "dataload", // 数据读入 + "train_launch", // 训练启动 + "train", // 训练 + "prepare", // 准备 + "proc", // 处理 + "model_format", // 模型格式转换 + "model_persistence", // 模型持久化 + "nodes_communication", // 节点间通信 + +#### 训练任务点事件规定字段 + "result", // 结果 + "reason", // 原因 + "average_power", // 平均功率 + "max_power", // 最大功率 + "resource_util_ratio", // 资源利用率 + "energy_consumption", // 能效 + "efficientcy", // 效率 + "accuracy" // 准确率 + +#### 推理任务持续事件规定字段 + "total", // 总体 + "e2e", // 端到端推理 + "query_sending", // 样本发送 + "result_sending", // 结果发送 + "dispatch", // 任务分派 + "preprocess", // 预处理 + "infer", // 推理 + "post", // 后处理 + "sample_process", // 样本处理 + "sample_dispatch_and_process", // 分派处理 + +#### 推理任务点事件规定字段 + "result", // 结果 + "reason", // 原因 + "latencyConstrainPercentile", // 超时比例 + "average_power", // 平均功率 + "max_power", // 最大功率 + "resource_util_ratio", // 资源利用率 + "energy_consumption", // 能效 + "efficientcy", // 效率 + "accuracy", // 准确率 + "elasticity", // 弹性 + "resilience_on_pressure" // 承压力 \ No newline at end of file diff --git a/ais_bench/__init__.py b/ais_bench/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..97a980704d9eadc8e5beb6c2344849f23db99aba --- /dev/null +++ b/ais_bench/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/ais_bench/logging/CMakeLists.txt b/ais_bench/logging/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..222f8c97ec9a7fb1fd17d4facf9992c6c5106918 --- /dev/null +++ b/ais_bench/logging/CMakeLists.txt @@ -0,0 +1,50 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 设置最小CMake版本 +cmake_minimum_required(VERSION 3.10) + +# 设置项目名和版本 +project(ais_bench_logging VERSION 1.0) + +# 设置C++标准 +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +set(CMAKE_CXX_FLAGS "-fPIC -std=c++11 -O0 -Wall -g2 ${CMAKE_CXX_FLAGS}") +add_compile_options(-Wno-format-extra-args) +add_compile_options(-fstack-protector-strong -Wl,-z,now -D_FORTIFY_SOURCE=2 -O2) +set(CMAKE_SHARED_LINKER_FLAGS "-fPIC -Wl,-z,relro,-z,now") +SET(CMAKE_SKIP_RPATH TRUE) + +# 添加源文件到项目中 +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/src DIR_SRCS) + +find_package(PythonLibs) +if( PYTHONLIBS_FOUND ) + include_directories(${PYTHON_INCLUDE_DIRS}) +else() + include_directories($ENV{PYTHON_INCLUDE_DIRS}) +endif() + +# 指定头文件路径 +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +link_directories(${CMAKE_CURRENT_SOURCE_DIR}/lib) + +# 定义SO库 +add_library(${PROJECT_NAME} SHARED ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/py_wrap.cpp) + +# 链接静态库 +target_link_libraries(${PROJECT_NAME} stdc++fs libtransmit.a) + diff --git a/ais_bench/logging/__init__.py b/ais_bench/logging/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..759225eb395715692c0f92a6b358e34e42aeea59 --- /dev/null +++ b/ais_bench/logging/__init__.py @@ -0,0 +1,15 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ais_bench.logging.libais_bench_logging import * \ No newline at end of file diff --git a/ais_bench/logging/include/Collector.h b/ais_bench/logging/include/Collector.h new file mode 100644 index 0000000000000000000000000000000000000000..e958b9f94b2c0b034357446207aa996683f2b242 --- /dev/null +++ b/ais_bench/logging/include/Collector.h @@ -0,0 +1,28 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "Global.h" +#include "Utils.h" + + +#ifndef COLLECTOR_H +#define COLLECTOR_H +class Collector { +public: + void Collect(const std::vector& files, json& dstJson); + +private: + LOGGER_RET MergeJsonFile(const std::string& filePath, json& mergedJson); +}; +#endif \ No newline at end of file diff --git a/ais_bench/logging/include/Global.h b/ais_bench/logging/include/Global.h new file mode 100644 index 0000000000000000000000000000000000000000..5ae364b469164c1c7c60970deaf6960c0481a77f --- /dev/null +++ b/ais_bench/logging/include/Global.h @@ -0,0 +1,43 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include "nlohmann/json.hpp" + +#ifndef GLOBAL_H +#define GLOBAL_H + +using json = nlohmann::json; +using LOGGER_RET = int; +using SET_RESULT_RET = int; + +enum { + SUCCESS = 0, + FAILED = 1, +}; + +enum class TaskType {UNDEFINED, TRAINING, INFERENCE}; + +enum class EventType {START, END, POINT}; + +struct DataPacket { + std::string eventName; + int sampleNum; + std::string value; + std::chrono::system_clock::time_point timePoint; +}; + +#endif \ No newline at end of file diff --git a/ais_bench/logging/include/Logger.h b/ais_bench/logging/include/Logger.h new file mode 100644 index 0000000000000000000000000000000000000000..26549d3209a4f14c3bd77ee248aee99714303d17 --- /dev/null +++ b/ais_bench/logging/include/Logger.h @@ -0,0 +1,74 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include "Global.h" +#include "Recorder.h" + +#ifndef LOGGER_H +#define LOGGER_H + + + +class Logger { +public: + static Logger* GetInstance(); + + void Init(const std::string& taskType, const std::string& logDirPath); + + void Start(const std::string& eventName, int sampleNum); + void End(const std::string& eventName, int sampleNum); + void Event(const std::string& eventName, std::string value); + + void Finish(); + + static LOGGER_RET CollectReport(const std::string task, const std::vector& logDirs); +private: + + static Logger* instance; + + Recorder m_recorder{TaskType::UNDEFINED}; + + bool m_initFlag = false; + bool m_saveResult = false; + TaskType m_task = TaskType::UNDEFINED; + std::string m_logDir; + std::ofstream m_logFileStream; + + std::mutex m_mtx; + + std::unordered_map m_taskHeaderMap = + {{TaskType::UNDEFINED, ""}, {TaskType::INFERENCE, "[INFERENCE] "}, {TaskType::TRAINING, "[TRAINING] "}}; + std::unordered_map m_eventHeaderMap = + {{EventType::POINT, ""}, {EventType::START, "[START] "}, {EventType::END, "[END] "}}; + + Logger() {} + ~Logger() { Deinit(); } + + void InitOnce(const std::string& logDirPath, const std::string& taskType); // can only be called by init() + void Deinit(); + + + void LogHelper(EventType eventType, DataPacket dataPacket); + DataPacket WrapData(std::string eventName, int sampleNum, std::string value); + +}; + + + +#endif // LOGGER_H \ No newline at end of file diff --git a/ais_bench/logging/include/Recorder.h b/ais_bench/logging/include/Recorder.h new file mode 100644 index 0000000000000000000000000000000000000000..ac607f21c246371759fd973268599d1fdb3e86af --- /dev/null +++ b/ais_bench/logging/include/Recorder.h @@ -0,0 +1,106 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include "Utils.h" +#include "Global.h" +#include "nlohmann/json.hpp" + +#ifndef RECORDER_H +#define RECORDER_H + +static const std::vector m_trainingDurationEventList = { + "dataload", // 数据读入 + "train_launch", // 训练启动 + "train", // 训练 + "prepare", // 准备 + "proc", // 处理 + "model_format", // 模型格式转换 + "model_persistence", // 模型持久化 + "nodes_communication", // 节点间通信 +}; +static const std::vector m_trainingPointEventList = { + "result", // 结果 + "reason", // 原因 + "average_power", // 平均功率 + "max_power", // 最大功率 + "resource_util_ratio", // 资源利用率 + "energy_consumption", // 能效 + "efficientcy", // 效率 + "accuracy" // 准确率 +}; +static const std::vector m_inferenceDurationEventList = { + "total", // 总体 + "e2e", // 端到端推理 + "query_sending", // 样本发送 + "result_sending", // 结果发送 + "dispatch", // 任务分派 + "preprocess", // 预处理 + "infer", // 推理 + "post", // 后处理 + "sample_process", // 样本处理 + "sample_dispatch_and_process", // 分派处理 +}; +static const std::vector m_inferencePointEventList = { + "result", // 结果 + "reason", // 原因 + "latencyConstrainPercentile", // 超时比例 + "average_power", // 平均功率 + "max_power", // 最大功率 + "resource_util_ratio", // 资源利用率 + "energy_consumption", // 能效 + "efficientcy", // 效率 + "accuracy", // 准确率 + "elasticity", // 弹性 + "resilience_on_pressure" // 承压力 +}; + +class Recorder { +public: + using DurationMapType = std::unordered_map; + using PointMapType = std::unordered_map; + Recorder(TaskType taskType); + void SetTaskType(TaskType taskType); + + void Record(EventType eventType, DataPacket dataPacket); + json GetOutJson(); + +private: + TaskType m_task = TaskType::UNDEFINED; + + std::vector m_durationEvent; + std::vector m_pointEvent; + + DurationMapType m_endRecording; + DurationMapType m_startRecording; + PointMapType m_pointRecording; + + void RecordEndHelper(DataPacket dataPacket); + void RecordStartHelper(DataPacket dataPacket); + void RecordPointHelper(DataPacket dataPacket); + + void ThroughputRatioToJson(const std::string& eventName, json& outJson); + void PointEventDataToJson(json& outJson); + void DurationEventDataToJson(json& outJson, DurationMapType& recording, std::string tailStr); + + void GetTrainingOutJson(json& outJson); + void GetInferenceOutJson(json& outJson); + + +}; + +#endif \ No newline at end of file diff --git a/ais_bench/logging/include/Utils.h b/ais_bench/logging/include/Utils.h new file mode 100644 index 0000000000000000000000000000000000000000..2805af47b6b15d9834a39c7e12bbf8f192402754 --- /dev/null +++ b/ais_bench/logging/include/Utils.h @@ -0,0 +1,45 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include "nlohmann/json.hpp" +#include "Global.h" + +#ifndef UTILS_H +#define UTILS_H +using json = nlohmann::json; + +namespace Utils +{ + bool HasDirectory(const std::string& path); + LOGGER_RET DeleteExistingFile(const std::string& logPath); + + std::string TimeToString(const std::chrono::system_clock::time_point& tp); + int64_t TimeToInt64Millisec(std::chrono::system_clock::time_point tp); + int CalculateMillisecondsDifference(const std::chrono::system_clock::time_point& t1, + const std::chrono::system_clock::time_point& t2); + + void writeStream(std::string text, std::ostream& out); + std::vector FindFiles(const std::vector &paths, std::regex pattern); + LOGGER_RET ReadJson(const std::string& filePath, json& jsonData); + + bool CheckFormat(const json& jsonData); +} // namespace Utils + + + +#endif \ No newline at end of file diff --git a/ais_bench/logging/include/ais_utils.h b/ais_bench/logging/include/ais_utils.h new file mode 100644 index 0000000000000000000000000000000000000000..f2cb0ffbb49ba4142142523be43aa00a63ecd550 --- /dev/null +++ b/ais_bench/logging/include/ais_utils.h @@ -0,0 +1,35 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef AISBENCH_UTILS_H +#define AISBENCH_UTILS_H +#include "Global.h" + +#ifdef __cplusplus +extern "C" { +#endif + + /// \brief 设置参数传递接口 传递相关参数值 + /// \arg + /// mode: 模式 training or inference + /// key: 设置项 详见 参数列表 + /// value: 设置的值 + /// backtrace: 当前为空 暂不需要传入 + SET_RESULT_RET set_result(const char* mode, const char* key, const char* value, const char* backtrace); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/ais_bench/logging/include/log.h b/ais_bench/logging/include/log.h new file mode 100644 index 0000000000000000000000000000000000000000..c9e0f524bb5e1963c71d6c47c2f86ff7237bc4e4 --- /dev/null +++ b/ais_bench/logging/include/log.h @@ -0,0 +1,126 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef LOG__H +#define LOG__H + +#include +#include +#include +#include +#include +#include +#include + + +#ifdef _WIN32 +#define TrimFilePath(x) strrchr(x, '\\') ? strrchr(x, '\\') + 1 : x +#else // Linux/Unix +#define TrimFilePath(x) strrchr(x, '/') ? strrchr(x, '/') + 1 : x +#endif + +#define LogColorNone_ "\033[0m" +#define LogColorRed_ "\033[0;31m" +#define LogColorLightRed_ "\033[1;31m" +#define LogColorYellow_ "\033[0;33m" +#define LogColorLightYellow_ "\033[1;33m" +#define LogColorGreen_ "\033[0;32m" +#define LogColorLightGreen_ "\033[1;32m" + +#define ColorfulPrint_(color, fmt, ...) \ + do { \ + printf(color fmt LogColorNone_, ##__VA_ARGS__); \ + } while (0) + +#define LOG_DEBUG 1 +#define LOG_INFO 2 +#define LOG_WARNING 3 +#define LOG_ERROR 4 + +#define LOG_DEBUG_STR "1" +#define LOG_INFO_STR "2" +#define LOG_WARNING_STR "3" +#define LOG_ERROR_STR "4" + +static std::string getCurrentTime() +{ + struct tm* local; + time_t now; + timeb tb; + ftime(&tb); + now = time(NULL); + local = localtime(&now); + char temp[50] = { 0 }; + int constNum = 1900; + sprintf(temp, "%d-%d-%d %d:%d:%d", local->tm_year + constNum, local->tm_mon + 1, local->tm_mday, + local->tm_hour, local->tm_min, local->tm_sec); + std::string timeNow = temp; + return timeNow; +} + +static int getLogLevel() +{ + int logLevel = LOG_INFO; + char* logLevelEnv = getenv("AISBENCH_LOG_LEVEL"); + if (logLevelEnv != nullptr) { + std::string logLevelStr = std::string(getenv("AISBENCH_LOG_LEVEL")); + if (logLevelStr == LOG_DEBUG_STR || logLevelStr == LOG_INFO_STR || + logLevelStr == LOG_WARNING_STR || logLevelStr == LOG_ERROR_STR) { + logLevel = std::stoi(logLevelStr); + } + } + return logLevel; +} + + +#define PrintLog_(color, mode, fmt, ...) \ + do { \ + ColorfulPrint_(color, "[%s][%s]" fmt "\n", \ + getCurrentTime().c_str(), std::string(mode).c_str(), ##__VA_ARGS__); \ + } while (0) + +#define PrintLog_debug(color, mode, fmt, ...) \ + do { \ + ColorfulPrint_(color, "[%s][%s][%s(%d)][%s()]: " fmt "\n", \ + getCurrentTime().c_str(), std::string(mode).c_str(), __FILE__, __LINE__, \ + __FUNCTION__, ##__VA_ARGS__); \ + } while (0) + + +static int FRIZY_LOG_LEVEL = getLogLevel(); + +#define PrintDbg_(mode, fmt, ...) PrintLog_debug(LogColorLightGreen_, mode, fmt, ##__VA_ARGS__) +#define PrintInfo_(mode, fmt, ...) PrintLog_(LogColorNone_, mode, fmt, ##__VA_ARGS__) +#define PrintWarn_(mode, fmt, ...) PrintLog_(LogColorLightYellow_, mode, fmt, ##__VA_ARGS__) +#define PrintErr_(mode, fmt, ...) PrintLog_(LogColorLightRed_, mode, fmt, ##__VA_ARGS__) + +#define LOG(level, fmt, ...) do { \ + if (level == LOG_DEBUG) { if (LOG_DEBUG >= FRIZY_LOG_LEVEL) {PrintDbg_("DEBUG", fmt, ##__VA_ARGS__);} \ + } else if (level == LOG_INFO) { if (LOG_INFO >= FRIZY_LOG_LEVEL) {PrintInfo_("INFO", fmt, ##__VA_ARGS__);} \ + } else if (level == LOG_WARNING) { if (LOG_WARNING >= FRIZY_LOG_LEVEL) {PrintWarn_("WARN", fmt, ##__VA_ARGS__);} \ + } else if (level == LOG_ERROR) { if (LOG_ERROR >= FRIZY_LOG_LEVEL) {PrintErr_("ERROR", fmt, ##__VA_ARGS__);} \ + } else { \ + } \ +} while (0) + + +#define SETLOGLEVEL(level) {FRIZY_LOG_LEVEL = level;} + +#define DEBUG(fmt, ...) LOG(LOG_DEBUG, fmt, ##__VA_ARGS__) +#define INFO(fmt, ...) LOG(LOG_INFO, fmt, ##__VA_ARGS__) +#define WARN(fmt, ...) LOG(LOG_WARNING, fmt, ##__VA_ARGS__) +#define ERROR(fmt, ...) LOG(LOG_ERROR, fmt, ##__VA_ARGS__) + + +#endif // LOG__H diff --git a/ais_bench/logging/py_wrap.cpp b/ais_bench/logging/py_wrap.cpp new file mode 100644 index 0000000000000000000000000000000000000000..4f7806b739cd817fdfce8d0c844ef9411d58334b --- /dev/null +++ b/ais_bench/logging/py_wrap.cpp @@ -0,0 +1,107 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "Logger.h" + +PyObject *Logger_Init(PyObject *self, PyObject *args) { + char *taskType; + char *logDirPath; + if (!PyArg_ParseTuple(args, "ss", &taskType, &logDirPath)) { + return NULL; + } + Logger::GetInstance()->Init(taskType, logDirPath); + Py_RETURN_NONE; +} +PyObject *Logger_Start(PyObject *self, PyObject *args) { + char *eventName; + int sampleNum; + if (!PyArg_ParseTuple(args, "si", &eventName, &sampleNum)) { + return NULL; + } + Logger::GetInstance()->Start(eventName, sampleNum); + Py_RETURN_NONE; +} +PyObject *Logger_End(PyObject *self, PyObject *args) { + char *eventName; + int sampleNum; + if (!PyArg_ParseTuple(args, "si", &eventName, &sampleNum)) { + return NULL; + } + Logger::GetInstance()->End(eventName, sampleNum); + Py_RETURN_NONE; +} +PyObject *Logger_Event(PyObject *self, PyObject *args) { + char *eventName; + char *value; + if (!PyArg_ParseTuple(args, "ss", &eventName, &value)) { + return NULL; + } + Logger::GetInstance()->Event(eventName, value); + Py_RETURN_NONE; +} +PyObject *Logger_CollectReport(PyObject *self, PyObject *args) { + char *task; + PyObject *logDirsObj; + if (!PyArg_ParseTuple(args, "sO", &task, &logDirsObj) || !PyList_Check(logDirsObj)) { + return NULL; + } + std::vector logDirs; + for (Py_ssize_t i = 0; i < PyList_Size(logDirsObj); i++) { + PyObject *item = PyList_GetItem(logDirsObj, i); + if (!PyUnicode_Check(item)) { + continue; + } + logDirs.push_back(PyUnicode_AsUTF8(item)); + } + int result = Logger::CollectReport(task, logDirs); + return PyLong_FromLong((long)result); +} +PyObject *Logger_Finish(PyObject *self, PyObject* Py_UNUSED(args)) { + Logger::GetInstance()->Finish(); + Py_RETURN_NONE; +} + + +static PyMethodDef LoggerMethods[] = { + {"init", Logger_Init, METH_VARARGS, "Initialize the Logger."}, + {"start", Logger_Start, METH_VARARGS, "Start logging an event."}, + {"end", Logger_End, METH_VARARGS, "End logging an event."}, + {"event", Logger_Event, METH_VARARGS, "Log an event with a value."}, + {"finish", Logger_Finish, METH_NOARGS, "Finish log and record."}, + {"collect_report", Logger_CollectReport, METH_VARARGS, "Collect data in log directories and report to stubs."}, + {NULL, NULL, 0, NULL} +}; + +PyDoc_STRVAR(ais_bench_logging_doc, "AISBench logging module"); + +static struct PyModuleDef ais_bench_logging_def = { + PyModuleDef_HEAD_INIT, + "ais_bench_logging", + ais_bench_logging_doc, + -1, + LoggerMethods, +}; + + +extern "C" { +PyMODINIT_FUNC PyInit_libais_bench_logging() { + PyObject* m = PyModule_Create(&ais_bench_logging_def); + if (!m) + { + return NULL; + } + return m; +} +} \ No newline at end of file diff --git a/ais_bench/logging/src/Collector.cpp b/ais_bench/logging/src/Collector.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b2f4bd9ae2835056b706e256b129b40b7eee5281 --- /dev/null +++ b/ais_bench/logging/src/Collector.cpp @@ -0,0 +1,51 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "Collector.h" +#include"log.h" + + +void Collector::Collect(const std::vector& files, json& dstJson) { + for (auto& file : files) { + if (MergeJsonFile(file, dstJson) == FAILED) { + DEBUG("collect performance data from file failed: %s", file.c_str()); + continue; + } + } +} + + +LOGGER_RET Collector::MergeJsonFile(const std::string& filePath, json& mergedJson) { + json jsonData; + if (Utils::ReadJson(filePath, jsonData) == FAILED) { + DEBUG("read json failed: %s", filePath.c_str()); + return FAILED; + } + if (!Utils::CheckFormat(jsonData)) { + DEBUG("json not in defined format: %s", filePath.c_str()); + return FAILED; + } + for (const auto& kv : jsonData.items()) { + auto& key = kv.key(); + auto& value = kv.value(); + if (mergedJson.find(key) == mergedJson.end() // new key + || (key.find("start") != std::string::npos && value["time"] < mergedJson[key]["time"]) // keep early + || (key.find("start") == std::string::npos && value["time"] > mergedJson[key]["time"])) // keep later record + { + mergedJson[key] = value; + } + } + return SUCCESS; +} \ No newline at end of file diff --git a/ais_bench/logging/src/Logger.cpp b/ais_bench/logging/src/Logger.cpp new file mode 100644 index 0000000000000000000000000000000000000000..ef959597d34bd6801cf72e3fa2ec2820c6be45b9 --- /dev/null +++ b/ais_bench/logging/src/Logger.cpp @@ -0,0 +1,177 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include "Logger.h" +#include "Global.h" +#include "Collector.h" +#include "ais_utils.h" +#include "Utils.h" +#include "log.h" + + +#define LOG_TITLE "[AISBENCH LOGGING]" +#define TRAIN_STR "training" +#define INFER_STR "inference" + +Logger* Logger::instance = nullptr; + +Logger* Logger::GetInstance() { + if (instance == nullptr) { + instance = new Logger(); + } + return instance; +} + +void Logger::Init(const std::string& taskType, const std::string& logDirPath) { + std::lock_guard lock(m_mtx); + if (!m_initFlag) { + InitOnce(logDirPath, taskType); + m_initFlag = true; + } +} + +void Logger::InitOnce(const std::string& logDirPath, const std::string& taskType) { + if (Utils::HasDirectory(logDirPath) && (taskType == TRAIN_STR || taskType == INFER_STR)) { + std::string logPath = logDirPath + "/" + std::to_string(getpid()) + "_" + taskType + ".log"; + if (Utils::DeleteExistingFile(logPath) == FAILED){ + WARN("delete existing log file failed: %s", logPath.c_str()); + return; + } + + m_logDir = logDirPath; + m_logFileStream.open(logPath); + m_task = taskType == TRAIN_STR ? TaskType::TRAINING : TaskType::INFERENCE; + m_saveResult = true; + m_recorder.SetTaskType(m_task); + } else { + WARN("log dir is invalid or taskType is not supported, init failed."); + } + return; +} + +void Logger::Finish() { + std::lock_guard lock(m_mtx); + Deinit(); +} + +void Logger::Deinit() { + if (m_saveResult) { + m_saveResult = false; + m_logFileStream.close(); + json outJson = m_recorder.GetOutJson(); + std::string typeStr = m_task == TaskType::TRAINING ? TRAIN_STR : INFER_STR; + std::string jsonPath = m_logDir + "/" + std::to_string(getpid()) + "_" + typeStr + ".json"; + if (Utils::DeleteExistingFile(jsonPath) == FAILED){ + WARN("delete existing json file failed: %s", jsonPath.c_str()); + return; + } + std::ofstream outFile(jsonPath); + if (outFile.is_open()) { + outFile << outJson; + outFile.close(); + } else { + WARN("open json file failed, cannot dump json : %s", jsonPath.c_str()); + } + } + return; +} + +DataPacket Logger::WrapData(std::string eventName, int sampleNum, std::string value) { + auto now = std::chrono::system_clock::now(); + DataPacket dataPacket = {eventName, sampleNum, value, now}; + return dataPacket; +} + + +void Logger::LogHelper(EventType eventType, DataPacket dataPacket) { + std::string text = LOG_TITLE; + + text += m_taskHeaderMap[m_task]; + text += Utils::TimeToString(dataPacket.timePoint) + " "; + text += m_eventHeaderMap[eventType]; + text += dataPacket.eventName + " "; + if (dataPacket.sampleNum > 0) { + text += "sample number: " + std::to_string(dataPacket.sampleNum); + } + if (dataPacket.value != "") { + text += "value: " + dataPacket.value; + } + + Utils::writeStream(text, std::cout); + if (m_saveResult) { + Utils::writeStream(text, m_logFileStream); + } +} + + +void Logger::Start(const std::string& eventName, int sampleNum) { + DataPacket dataPacket = WrapData(eventName, sampleNum, ""); + std::lock_guard lock(m_mtx); + LogHelper(EventType::START, dataPacket); + m_recorder.Record(EventType::START, dataPacket); +} + +void Logger::End(const std::string& eventName, int sampleNum) { + DataPacket dataPacket = WrapData(eventName, sampleNum, ""); + std::lock_guard lock(m_mtx); + LogHelper(EventType::END, dataPacket); + m_recorder.Record(EventType::END, dataPacket); +} + +void Logger::Event(const std::string& eventName, std::string value) { + DataPacket dataPacket = WrapData(eventName, 0, value); + std::lock_guard lock(m_mtx); + LogHelper(EventType::POINT, dataPacket); + m_recorder.Record(EventType::POINT, dataPacket); +} + +LOGGER_RET Logger::CollectReport(const std::string task, const std::vector& logDirs) { + if (task != TRAIN_STR && task != INFER_STR) { + WARN("task name should be training or inference"); + return FAILED; + } + std::regex pattern("^[1-9][0-9]*_" + task + ".json$"); //todo modify the pattern + std::vector files = Utils::FindFiles(logDirs, pattern); //check file path in FindFiles todo + if (files.size() == 0) { + WARN("no performance data founded."); + return FAILED; + } + + // collect all file into one json + json jsonData; + Collector collector = Collector(); + collector.Collect(files, jsonData); + + // report this json to stubs + if (!Utils::CheckFormat(jsonData)) { + WARN("collected josn data not in defined format."); + return FAILED; + } else { + for (const auto& kv : jsonData.items()) { + auto& key = kv.key(); + std::string value = kv.value()["value"]; + if (set_result(task.c_str(), key.c_str(), value.c_str(), "[]") == FAILED) { + WARN("set result failed, mode: %s, key: %s, value: %s", task.c_str(), key.c_str(), value.c_str()); + return FAILED; + } + } + } + return SUCCESS; +} diff --git a/ais_bench/logging/src/Recorder.cpp b/ais_bench/logging/src/Recorder.cpp new file mode 100644 index 0000000000000000000000000000000000000000..35f388027f6f7819cb673aaedc5725be79c14722 --- /dev/null +++ b/ais_bench/logging/src/Recorder.cpp @@ -0,0 +1,149 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "Recorder.h" + + + + +Recorder::Recorder(TaskType taskType) { + m_task = taskType; + SetTaskType(taskType); +} + +void Recorder::SetTaskType(TaskType taskType) { + if (m_task != taskType) { + m_endRecording.clear(); + m_startRecording.clear(); + m_pointRecording.clear(); + m_task = taskType; + } + if (taskType == TaskType::INFERENCE) { + m_durationEvent = m_inferenceDurationEventList; + m_pointEvent = m_inferencePointEventList; + } else if (taskType == TaskType::TRAINING) { + m_durationEvent = m_trainingDurationEventList; + m_pointEvent = m_trainingPointEventList; + } +} + +void Recorder::Record(EventType eventType, DataPacket dataPacket) { + if (m_task == TaskType::UNDEFINED) { + return; + } + if (eventType == EventType::END) { + RecordEndHelper(dataPacket); + } else if (eventType == EventType::START) { + RecordStartHelper(dataPacket); + } else { + RecordPointHelper(dataPacket); + } +} + +void Recorder::RecordEndHelper(DataPacket dataPacket) { + std::string event = dataPacket.eventName; + if (m_startRecording.find(event) != m_startRecording.end() && // 在start表中 + m_startRecording[event].sampleNum == dataPacket.sampleNum && // sample数匹配 + m_startRecording[event].timePoint < dataPacket.timePoint) { // 时间在start之后 + m_endRecording[event] = dataPacket; + } +} + +void Recorder::RecordStartHelper(DataPacket dataPacket) { + std::string event = dataPacket.eventName; + if (std::find(m_durationEvent.begin(), m_durationEvent.end(), event) != m_durationEvent.end() && // 是规定字段 + m_startRecording.find(event) == m_startRecording.end()) { // 未被记录 + m_startRecording[event] = dataPacket; + } +} + +void Recorder::RecordPointHelper(DataPacket dataPacket) { + std::string event = dataPacket.eventName; + if (std::find(m_pointEvent.begin(), m_pointEvent.end(), event) != m_pointEvent.end()) { // 是规定字段 + m_pointRecording[event] = dataPacket; + } +} + +void Recorder::ThroughputRatioToJson(const std::string& eventName, json& outJson) { + if (m_startRecording.find(eventName) != m_startRecording.end() // exist in start recording + && m_endRecording.find(eventName) != m_endRecording.end() // exist in end recording + && m_startRecording[eventName].sampleNum == m_endRecording[eventName].sampleNum) { // sample number match + int64_t endMillisec = Utils::TimeToInt64Millisec(m_endRecording[eventName].timePoint); + int timeMilliDiff = Utils::CalculateMillisecondsDifference(m_startRecording[eventName].timePoint, + m_endRecording[eventName].timePoint); + int sampleNumber = m_startRecording[eventName].sampleNum; + double throughtputRatio = sampleNumber * 1000.0 / timeMilliDiff; + outJson["throughput_ratio"] = json({{"value", std::to_string(throughtputRatio)}, {"time", endMillisec}}); + } +} + +void Recorder::PointEventDataToJson(json& outJson) { + for (auto& kv : m_pointRecording) { + auto& eventName = kv.first; + DataPacket& dataPacket = kv.second; + int64_t millisec = Utils::TimeToInt64Millisec(dataPacket.timePoint); + outJson[eventName] = json({{"value", dataPacket.value}, {"time", millisec}}); + } +} + +void Recorder::DurationEventDataToJson(json& outJson, DurationMapType& recording, std::string tailStr) { + for (auto&kv : recording) { + auto& eventName = kv.first; + DataPacket& dataPacket = kv.second; + std::string timeStr = Utils::TimeToString(dataPacket.timePoint); + int64_t millisec = Utils::TimeToInt64Millisec(dataPacket.timePoint); + outJson[eventName + tailStr] = json({{"value", timeStr}, {"time", millisec}}); + } +} + +void Recorder::GetTrainingOutJson(json& outJson) { + // 1. get point event data + PointEventDataToJson(outJson); + // 2. get start event data + DurationEventDataToJson(outJson, m_startRecording, "_start_time"); + // 3. get end event data + DurationEventDataToJson(outJson, m_endRecording, "_end_time"); + // 4. compute throughtput ratio use train duration event record + ThroughputRatioToJson("train", outJson); +} + +void Recorder::GetInferenceOutJson(json& outJson) { + // 1. get point event data + PointEventDataToJson(outJson); + // 2. calculate latency + for (auto&kv : m_startRecording) { + auto& eventName = kv.first; + if (m_endRecording.find(eventName) != m_endRecording.end() // event name match + && m_startRecording[eventName].sampleNum == m_endRecording[eventName].sampleNum) { // sample number match + int64_t endMillisec = Utils::TimeToInt64Millisec(m_endRecording[eventName].timePoint); + int timeMilliDiff = Utils::CalculateMillisecondsDifference(m_startRecording[eventName].timePoint, + m_endRecording[eventName].timePoint); + double latency = timeMilliDiff / 1000.0; + outJson[eventName + "_latency"] = json({{"value", std::to_string(latency)}, {"time", endMillisec}}); + } + } + // 3. calculate throughput + ThroughputRatioToJson("infer", outJson); +} + +json Recorder::GetOutJson() { + json outJson; + if (m_task == TaskType::INFERENCE) { + GetInferenceOutJson(outJson); + } else if (m_task == TaskType::TRAINING) { + GetTrainingOutJson(outJson); + } + return outJson; +} \ No newline at end of file diff --git a/ais_bench/logging/src/Utils.cpp b/ais_bench/logging/src/Utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..bada354c9c0fe308ea25510358e3c056f6f38641 --- /dev/null +++ b/ais_bench/logging/src/Utils.cpp @@ -0,0 +1,169 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "nlohmann/json.hpp" +#include "Global.h" +#include "log.h" +#include "Utils.h" + + +namespace Utils +{ + void writeStream(std::string text, std::ostream& out) { + text += "\n"; + out << text; + } + + LOGGER_RET CreateDir(const std::string& path) { + try { + std::experimental::filesystem::create_directories(path); + DEBUG("new directories has been created: %s", path.c_str()); + } catch (const std::experimental::filesystem::filesystem_error& e) { + DEBUG("creating new directories failed: %s", path.c_str()); + return FAILED; + } + return SUCCESS; + } + + bool IsDirectory(const std::string& path) { + return std::experimental::filesystem::exists(path) && std::experimental::filesystem::is_directory(path); + } + + bool HasDirectory(const std::string& path) { + if (IsDirectory(path)) { + return true; + } else { + if (CreateDir(path) == FAILED) { + return false; + } + } + return true; + } + + LOGGER_RET DeleteExistingFile(const std::string& logPath) { + if (std::experimental::filesystem::exists(logPath)) { + try { + std::experimental::filesystem::remove(logPath); + DEBUG("existing log file been deleted successfully: %s", logPath.c_str()); + } catch (const std::system_error& e) { + DEBUG("delete existing log file failed due to reason: %s", e.what()); + return FAILED; + } + } + return SUCCESS; + } + + std::vector FindFilesHelper(const std::string& path, std::regex pattern) { + std::experimental::filesystem::path p(path); + std::vector result; + if (!std::experimental::filesystem::exists(p)) { + DEBUG("path not exists skip it: %s", path.c_str()); + return result; + } + + if (std::experimental::filesystem::is_directory(p)) { + for (auto& entry : std::experimental::filesystem::directory_iterator(p)) { + std::vector resultRecursive = FindFilesHelper(entry.path().string(), pattern); + result.insert(result.end(), resultRecursive.begin(), resultRecursive.end()); + } + } else if (std::regex_match(p.filename().string(), pattern)) { + result.emplace_back(p.string()); + } + return result; + } + + std::vector FindFiles(const std::vector &paths, std::regex pattern) { + std::vector result; + for (auto& path : paths) { + auto resTmp = FindFilesHelper(path, pattern); + result.insert(result.end(), resTmp.begin(), resTmp.end()); + } + return result; + } + + LOGGER_RET ReadJson(const std::string& filePath, json& jsonData) { + std::ifstream file(filePath); + if (!file.is_open()) { + DEBUG("open json file failed: %s", filePath.c_str()); + return FAILED; + } + try { + file >> jsonData; + } catch (json::parse_error &e) { + DEBUG("parse json file failed, ret: %s", e.what()); + file.close(); + return FAILED; + } + file.close(); + return SUCCESS; + } + + bool CheckFormat(const json& jsonData) { + if (!jsonData.is_object()) { + return false; + } + for (const auto& kv : jsonData.items()) { + auto& value = kv.value(); + bool checkValue = value.is_object() && value.contains("value") && value["value"].is_string() + && value.contains("time") && value["time"].is_number_integer(); + if (!checkValue) { + return false; + } + } + return true; + } + + std::string TimeToString(const std::chrono::system_clock::time_point& tp) { + std::time_t tt = std::chrono::system_clock::to_time_t(tp); + std::tm* tm_prt = std::localtime(&tt); + + std::stringstream ss; + ss << std::put_time(tm_prt, "%Y-%m-%d"); + ss << " "; + ss << std::put_time(tm_prt, "%H:%M:%S"); + ss << "."; + + auto milliseconds = + std::chrono::duration_cast(tp.time_since_epoch()) % 1000; + ss << std::setfill('0') << std::setw(3) << milliseconds.count(); + + return ss.str(); + } + + int64_t TimeToInt64Millisec(std::chrono::system_clock::time_point tp) { + auto microseconds = std::chrono::time_point_cast(tp); + auto epoch_microseconds = microseconds.time_since_epoch(); + int64_t milliseconds = static_cast(epoch_microseconds.count()) / 1000; + + return milliseconds; + } + + int CalculateMillisecondsDifference(const std::chrono::system_clock::time_point& t1, + const std::chrono::system_clock::time_point& t2) { + std::chrono::duration duration = + std::chrono::duration_cast(t2 - t1); + return duration.count(); + } + +} // namespace Utils + diff --git a/ais_bench/logging/src/ais_utils.cpp b/ais_bench/logging/src/ais_utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..64a793d5b06aacc74aa67b22b9a5fa3772845cec --- /dev/null +++ b/ais_bench/logging/src/ais_utils.cpp @@ -0,0 +1,226 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "transmit/transmit_client.h" +#include "Global.h" +using namespace std; + +// 训练支持参数 +char const* training_list[] = { "result", "reason", // 结果 原因 + "total_start_time", "total_end_time", // 总体训练开始时间 总体训练结束时间 + "prepare_start_time", "prepare_end_time", // 准备开始时间 准备结束时间 + "proc_start_time", "proc_end_time", // 处理开始时间 总处理结束时间 + "dataload_start_time", "dataload_end_time", // 数据读入开始时间 数据读入结束时间 + "train_launch_start_time", "train_launch_end_time", // 训练启动开始时间 训练结束开始时间 + "train_start_time", "train_end_time", // 训练开启时间 训练结束时间 + "model_format_start_time", "model_format_end_time", // 模型格式转换开始时间 模型格式转换结束时间 + "model_persistence_start_time", "model_persistence_end_time", // 模型持久化开始时间 模型持久化结束时间 + "nodes_communication_start_time", "nodes_communication_end_time", // 节点间通信时延 开始时间 结束时间 + "average_power", "max_power", // 平均功率 最大功率 + "throughput_ratio", "resource_util_ratio", // 吞吐率 资源利用率 + "energy_consumption", "efficientcy", "accuracy" // 能效 效率 准确率 +}; +// 推理支持参数 +char const* inference_list[] = { "result", "reason", // 结果 原因 + "total_latency", // 推理总延时 + "e2e_latency", // 端到端推理延时 + "query_sending_latency", // 样本发送延时 + "result_sending_latency", // 结果发送延时 + "dispatch_latency", // 任务分派延时 + "preprocess_latency", // 预处理延时 + "infer_latency", // 推理延时 + "post_latency", // 后处理延时 + "sample_process_latency", // 样本处理延时 + "latencyConstrainPercentile", // 超时比例 + "sample_dispatch_and_process_latency", // 分派处理延时 + "average_power", "max_power", // 平均功率 最大功率 + "throughput_ratio", "resource_util_ratio", // 吞吐率 资源利用率 + "energy_consumption", "efficientcy", "accuracy", // 能效 效率 准确率 + "elasticity", // 弹性 + "resilience_on_pressure" // 承压力 +}; + +string GetKeyValue(std::string sKey, std::string sValue) +{ + char szDoubleQutoes[] = "\""; + char szColon[] = ":"; + std::string strResult; + strResult.append(szDoubleQutoes); + strResult.append(sKey); + strResult.append(szDoubleQutoes); + strResult.append(szColon); + strResult.append(szDoubleQutoes); + strResult.append(sValue); + strResult.append(szDoubleQutoes); + return strResult; +} + +int GetCmdLineFromPid(int pid, string& outcmdline) +{ + char path[PATH_MAX + 1] = { 0 }; + snprintf(path, sizeof(path), "/proc/%i/cmdline", pid); + char buffer[256] = { 0 }; + FILE* pFile = fopen(path, "r"); + if (pFile == nullptr) { + return -1; + } + int iNumber = 255; + fread(buffer, sizeof(char), iNumber, pFile); + for (uint64_t i = 0; i < sizeof(buffer) - 1; i++) { + if (buffer[i] == 0 && buffer[i + 1] == 0) { + break; + } + if (buffer[i] == 0) { + buffer[i] = ' '; + } + } + outcmdline.append(buffer); + fclose(pFile); + return 0; +} + +vector splitStr(string str, char delimiter) +{ + vector r; + while (!str.empty()) { + int ind = str.find_first_of(delimiter); + if (ind == -1) { + r.push_back(str); + str.clear(); + } else { + r.push_back(str.substr(0, ind)); + str = str.substr(ind + 1, str.size() - ind - 1); + } + } + return r; +} + +string GetCmdResult(const string& strCmd) +{ + char buf[10240] = { 0 }; + FILE* pf = nullptr; + if ((pf = popen(strCmd.c_str(), "r")) == nullptr) { + return ""; + } + string strResult; + while (fgets(buf, sizeof(buf), pf)) { + strResult += buf; + } + pclose(pf); + unsigned int iSize = strResult.size(); + if (iSize > 0 && strResult[iSize - 1] == '\n') { + strResult = strResult.substr(0, iSize - 1); + } + return splitStr(strResult, ' ')[1].c_str(); +} + +string StrToJson(string strkey, string strvalue, string strbacktrace, pid_t pid, \ + pid_t pgid, string outcmdlinestr, string working_path) +{ + string str("{\""); + str.append(strkey); + str.append("\":{"); + str.append(GetKeyValue("value", strvalue)); + str.append(","); + str.append("\"backtrace\":"); + str.append(strbacktrace); + str.append(","); + str.append(GetKeyValue("pid", to_string(pid))); + str.append(","); + str.append("\"gid\":"); + str.append(to_string(pgid)); + str.append(","); + str.append(GetKeyValue("cmdline", outcmdlinestr)); + str.append(","); + str.append(GetKeyValue("working_path", working_path)); + str.append("}"); + str.append("}"); + return str; +} + +SET_RESULT_RET set_result_inner(string strkey, string strvalue, string strbacktrace) +{ + pid_t pid, pgid; + pid = getpid(); + pgid = getpgid(0); + string outcmdlinestr; + GetCmdLineFromPid(pid, outcmdlinestr); + string pwdstr = "pwdx " + to_string(pid); + string working_path = GetCmdResult(pwdstr); + string str = StrToJson(strkey, strvalue, strbacktrace, pid, pgid, outcmdlinestr, working_path); + // 调用上传stub的接口 + Transmit_client transmit = Transmit_client(); + int ret = transmit.SentMsgToServer(str); + if (ret != 0) { + printf("set result error sendmsg to Server failed:%d\n", ret); + return FAILED; + } + return SUCCESS; +} + + + +int check_mode_key_valid(const char* mode, const char* key) +{ + int valid = false; + if (strcmp(mode, "training") == 0) { + for (uint64_t i = 0; i < sizeof(training_list) / sizeof(training_list[0]); i++) { + if (strcmp(key, training_list[i]) == 0) { + valid = true; + break; + } + } + } else if (strcmp(mode, "inference") == 0) { + for (uint64_t i = 0; i < sizeof(inference_list) / sizeof(inference_list[0]); i++) { + if (strcmp(key, inference_list[i]) == 0) { + valid = true; + break; + } + } + } else { + printf("not match model,please check the model:%s!!!!\n", mode); + return false; + } + if (valid == false) { + printf("not find mode:%s key:%s\n", mode, key); + } + return valid; +} + + +// 接口函数 +#ifdef __cplusplus +extern "C" { +#endif + + SET_RESULT_RET set_result(const char* mode, const char* key, const char* value, const char* backtrace) + { + if (check_mode_key_valid(mode, key) == false) { + return FAILED; + } + return set_result_inner(key, value, backtrace); + } + +#ifdef __cplusplus +} +#endif diff --git a/build.sh b/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..06461a8b830970034cfa01ea42a2c6b6c75746b9 --- /dev/null +++ b/build.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CUR_PATH=$(cd "$(dirname "$0")";pwd) +TRANSMIT_DIR=${CUR_PATH}/dependencies/transmit +LOGGING_SRC_PATH=${CUR_PATH}/ais_bench/logging +TEST_PATH=${CUR_PATH}/test + +ret_ok=0 +ret_failed=1 + + +build_transmit() { + bash -x ${TRANSMIT_DIR}/build.sh + cp ${TRANSMIT_DIR}/output/include/* -rf ${LOGGING_SRC_PATH}/include + cp ${TRANSMIT_DIR}/output/lib -rf ${LOGGING_SRC_PATH} +} + +prepare_nlohmann_json() { + cd ${CUR_PATH} + [ -d ${CUR_PATH}/json ] && rm -rf ${CUR_PATH}/json + git clone https://github.com/nlohmann/json.git + ret=$? + if [ $ret != $ret_ok ];then + echo "download nlohmann json failed ret:$ret" + exit $ret_failed + fi + echo "download nlohmann json successfully ret:$ret" + cp ${CUR_PATH}/json/single_include/* -rf ${LOGGING_SRC_PATH}/include +} + +prepare_gtest() { + cd ${TEST_PATH} + [ -d ${TEST_PATH}/googletest ] && rm -rf ${TEST_PATH}/googletest + git clone https://github.com/google/googletest.git + ret=$? + if [ $ret != $ret_ok ];then + echo "download gtest failed ret:$ret" + exit $ret_failed + fi + echo "download gtest successfully ret:$ret" + cd googletest + cmake . + make install + +} + +build_logging() { + [ -d ${LOGGING_SRC_PATH}/build ] && rm -rf ${LOGGING_SRC_PATH}/build + mkdir -p ${LOGGING_SRC_PATH}/build + cd ${LOGGING_SRC_PATH}/build + cmake .. + make + cp ${LOGGING_SRC_PATH}/build/*.so ${LOGGING_SRC_PATH} + + cd ${CUR_PATH} + python3 setup.py bdist_wheel + [ -d ${CUR_PATH}/output ] && rm -rf ${CUR_PATH}/output + mkdir ${CUR_PATH}/output + cp ${CUR_PATH}/dist/*.whl ${CUR_PATH}/output + cp ${CUR_PATH}/README.md ${CUR_PATH}/output +} + + +build_test() { + [ -d ${TEST_PATH}/build ] && rm -rf ${TEST_PATH}/build + mkdir -p ${TEST_PATH}/build + cd ${TEST_PATH}/build + cmake .. + make +} + +build_transmit +prepare_nlohmann_json +build_logging + +if [ "$1" == "full" ];then + prepare_gtest + build_test +fi \ No newline at end of file diff --git a/dependencies/transmit/CMakeLists.txt b/dependencies/transmit/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..c443d08c0bdc98d0b49e23b09befdac20597eae7 --- /dev/null +++ b/dependencies/transmit/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.10) +project(transmit) + +set(CMAKE_CXX_FLAGS "-fPIC -std=c++11 -O0 -Wall -g2 ${CMAKE_CXX_FLAGS}") +set(CMAKE_C_FLAGS "-fPIC ${CMAKE_C_FLAGS}") + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${CMAKE_CURRENT_SOURCE_DIR}/src) +add_library(${PROJECT_NAME} STATIC + src/aesutils/aesutils.cpp + src/tiny-AES-c/aes.c + src/transmit_server.cpp + src/transmit_client.cpp) + +target_link_libraries(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) \ No newline at end of file diff --git a/dependencies/transmit/README.md b/dependencies/transmit/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9cbfc8aa99d837afedca12a9f844683fef71c711 --- /dev/null +++ b/dependencies/transmit/README.md @@ -0,0 +1,3 @@ +## intro + +### This module is used by logging module and stubs module to transmit infer/train performance data. \ No newline at end of file diff --git a/dependencies/transmit/build.sh b/dependencies/transmit/build.sh new file mode 100644 index 0000000000000000000000000000000000000000..a28d5cc48d859e2f2c61b105c469aacbe89c8a2e --- /dev/null +++ b/dependencies/transmit/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +WORK_PATH=$(cd "$(dirname "$0")";pwd) +BUILD_DIR=${WORK_PATH}/build +OUTPUT_DIR=${WORK_PATH}/output + +# build transmit +[ -d ${BUILD_DIR} ] && rm -rf ${BUILD_DIR} +mkdir ${BUILD_DIR} +cd ${BUILD_DIR} +cmake .. +make + +# move to output dir +# output +# ├─include +# │ ├─transmit +# │ │ ├─transmit_client.h +# │ │ └─transmit_server.h +# └─lib +# └─libtransmit.a +[ -d ${OUTPUT_DIR} ] && rm -rf ${OUTPUT_DIR} +mkdir -p ${OUTPUT_DIR}/include/ +mkdir -p ${OUTPUT_DIR}/lib +cp ${BUILD_DIR}/lib*.a ${OUTPUT_DIR}/lib +cp ${WORK_PATH}/include/* -rf ${OUTPUT_DIR}/include/ + +#clean +rm -rf ${BUILD_DIR} \ No newline at end of file diff --git a/dependencies/transmit/include/transmit/transmit_client.h b/dependencies/transmit/include/transmit/transmit_client.h new file mode 100644 index 0000000000000000000000000000000000000000..aade245057af0e158f3bdd9f2611771e1ad6255e --- /dev/null +++ b/dependencies/transmit/include/transmit/transmit_client.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) Aisbench. 2020-2022. All rights reserved. + * description: + * + */ + +#ifndef TRANSMIT_CLIENT_H +#define TRANSMIT_CLIENT_H + +#include +#include + +class Transmit_client { +public: + Transmit_client(); + ~Transmit_client(); + + int SentMsgToServer(const std::string& msg); + +private: + std::string aeskey; + std::string salt; + bool isInited; + bool initialization(); + static std::string generateTransmitMSG(std::string msg); +}; + +#endif // TRANSMIT_CLIENT_H diff --git a/dependencies/transmit/include/transmit/transmit_server.h b/dependencies/transmit/include/transmit/transmit_server.h new file mode 100644 index 0000000000000000000000000000000000000000..6fca7ea4c5f7640800dee02e5a6403e462fd839d --- /dev/null +++ b/dependencies/transmit/include/transmit/transmit_server.h @@ -0,0 +1,62 @@ +/* + * Copyright (c) Aisbench. 2020-2022. All rights reserved. + * Description: transmit_server.h + * Author: + * Create: + */ + +#ifndef TRANSMIT_SERVER_H +#define TRANSMIT_SERVER_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const uint64_t DEFAULT_TAG = 0x01020304; +const uint64_t MAX_LENGTH = 0x00100000; // 1M +#pragma pack(1) +typedef struct { + uint32_t tag; + uint32_t len; + uint8_t* buf; +} Tlv; +#pragma pack() + +class Transmit_server { +public: + using callbackProc = std::function; + Transmit_server(); + ~Transmit_server(); + int Start(); + int Worker(); + int Stop(); + void setCallbackMsgProc(callbackProc cb) + { + callBack = cb; + } +private: + bool StopTransmitThread(); + int ConnectInit(); + pthread_t transmitThreadID; + int PostprocessData(uint8_t* input, uint32_t length); + void SignalHandler(int signum); + int ReceiveMsg(); + void CleanTCPResource(); + bool isHealthy; + std::string aeskey; + std::string salt; + int ser_sock = -1; + int clnt_sock; + std::string ReceiveMsgStr; + std::string AfterDecode; + Tlv frame; + callbackProc callBack; +}; + +#endif // TRANSMIT_SERVER_H diff --git a/dependencies/transmit/src/aesutils/CMakeLists.txt b/dependencies/transmit/src/aesutils/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..37887612d5df1493340b6eee8c11cf0bf0ffd476 --- /dev/null +++ b/dependencies/transmit/src/aesutils/CMakeLists.txt @@ -0,0 +1,3 @@ +aux_source_directory(. DIR_AES_UTIL_SRCS) +set(CMAKE_CXX_FLAGS "-fPIC -std=c++11 -O0 -Wall -g2 ${CMAKE_CXX_FLAGS}") +add_library(aseutil STATIC ${DIR_AES_UTIL_SRCS}) \ No newline at end of file diff --git a/dependencies/transmit/src/aesutils/aesutils.cpp b/dependencies/transmit/src/aesutils/aesutils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..6eea9576d4db1a5e743fde5fca96a36333a653ef --- /dev/null +++ b/dependencies/transmit/src/aesutils/aesutils.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) Aisbench. 2020-2022. All rights reserved. + * description: + * + */ + +#include "aesutils.h" +#include +#include + +AesUtils::AesUtils() +{ +} + +AesUtils::~AesUtils() +{ +} + +int AesUtils::encrypt_cbc_256(uint8_t* inputStr, int length) +{ + struct AES_ctx ctx; + AES_init_ctx_iv(&ctx, (const uint8_t*)AESPUBLICKEY, (const uint8_t*)SALT); + AES_CBC_encrypt_buffer(&ctx, inputStr, length); + return 0; +} + +int AesUtils::decrypt_cbc_256(uint8_t* inputStr, int length) +{ + struct AES_ctx ctx; + + AES_init_ctx_iv(&ctx, (const uint8_t*)AESPUBLICKEY, (const uint8_t*)SALT); + AES_CBC_decrypt_buffer(&ctx, inputStr, length); + return 0; +} + +std::string AesUtils::appendStrWithZero(std::string inputStr, int length) +{ + std::string retStr = ""; + if (inputStr.empty() || 0 == inputStr.length()) { + return retStr; + } + + retStr = inputStr; + int strLen = inputStr.length(); // "0"的长度 = 指定的长度 - 当前的长度。 + if (strLen == length) { + return retStr; + } else if (strLen < length) { + int temp = length - strLen; + retStr.append(temp, '0'); + return retStr; + } else { + return inputStr.substr(0, length); + } +} + +std::string generateRandomPublicAESKey() +{ + return "89Hjufa-Efafhhk6"; +} diff --git a/dependencies/transmit/src/aesutils/aesutils.h b/dependencies/transmit/src/aesutils/aesutils.h new file mode 100644 index 0000000000000000000000000000000000000000..0e83e7925c8ea40bc6e87de352325edce08a0c33 --- /dev/null +++ b/dependencies/transmit/src/aesutils/aesutils.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) Aisbench. 2020-2022. All rights reserved. + * Description: aesutils.h + * Author: + * Create: + */ + +#ifndef AESUTILS_H +#define AESUTILS_H +#include +#include "tiny-AES-c/aes.hpp" +#include +#include +#include "global.h" + + // 定义程序中使用的常量 +#define SERVER_ADDRESS "127.0.0.1" // 服务器端IP地址 +#define MSGSIZE 1024 // 收发缓冲区的大小 +#define SERVERPORT 9990 +#define BLOCK_SIZE 16 + +class AesUtils { +public: + AesUtils(); + ~AesUtils(); + static int + encrypt_cbc_256(uint8_t* inputStr, int length); + static int + decrypt_cbc_256(uint8_t* inputStr, int length); + static std::string generateRandomPublicAESKey(); +private: + static std::string appendStrWithZero(std::string inputStr, int length); +}; + +#endif // AESUTILS_H diff --git a/dependencies/transmit/src/aesutils/global.h b/dependencies/transmit/src/aesutils/global.h new file mode 100644 index 0000000000000000000000000000000000000000..1ccd2c2ba6ebf9789065fea0de41fd9af0f039ab --- /dev/null +++ b/dependencies/transmit/src/aesutils/global.h @@ -0,0 +1,14 @@ +/* + * Copyright (c) Aisbench. 2020-2022. All rights reserved. + * Description: aesutils.h + * Author: + * Create: + */ + +#ifndef GLOBAL_H +#define GLOBAL_H +#define AESPUBLICKEY "2b7e151628aed2a6abf7158809cf4f3c" +#define SALT "hellgoodwellport" + + +#endif // GLOBAL_H diff --git a/dependencies/transmit/src/log.h b/dependencies/transmit/src/log.h new file mode 100644 index 0000000000000000000000000000000000000000..ebbdb38fb5be381ebfa5bcd0f8c3938281dd9226 --- /dev/null +++ b/dependencies/transmit/src/log.h @@ -0,0 +1,115 @@ +/* + * Copyright (c) Aisbench. 2020-2022. All rights reserved. + * Description: + * Author: + * Create: + */ + +#ifndef LOG__H +#define LOG__H + +#include +#include +#include +#include +#include +#include + + +#ifdef _WIN32 +#define TrimFilePath(x) strrchr(x, '\\') ? strrchr(x, '\\') + 1 : x +#else // Linux/Unix +#define TrimFilePath(x) strrchr(x, '/') ? strrchr(x, '/') + 1 : x +#endif + +#define LogColorNone_ "\033[0m" +#define LogColorRed_ "\033[0;31m" +#define LogColorLightRed_ "\033[1;31m" +#define LogColorYellow_ "\033[0;33m" +#define LogColorLightYellow_ "\033[1;33m" +#define LogColorGreen_ "\033[0;32m" +#define LogColorLightGreen_ "\033[1;32m" + +#define ColorfulPrint_(color, fmt, ...) \ + do { \ + printf(color fmt LogColorNone_, ##__VA_ARGS__); \ + } while (0) + +static std::string getCurrentTime() +{ + struct tm* local; + time_t now; + timeb tb; + ftime(&tb); + now = time(NULL); + local = localtime(&now); + char temp[50] = { 0 }; + int constNum = 1900; + sprintf(temp, "%d-%d-%d %d:%d:%d", local->tm_year + constNum, local->tm_mon + 1, local->tm_mday, + local->tm_hour, local->tm_min, local->tm_sec); + std::string timeNow = temp; + return timeNow; +} + +#define PrintLog_(color, mode, fmt, ...) \ + do { \ + ColorfulPrint_(color, "[%s][%s]" fmt "\n", \ + getCurrentTime().c_str(), std::string(mode).c_str(), ##__VA_ARGS__); \ + } while (0) + +#define PrintLog_debug(color, mode, fmt, ...) \ + do { \ + ColorfulPrint_(color, "[%s][%s][%s(%d)][%s()]: " fmt "\n", \ + getCurrentTime().c_str(), std::string(mode).c_str(), __FILE__, __LINE__, \ + __FUNCTION__, ##__VA_ARGS__); \ + } while (0) + +#define LOG_DEBUG 1 +#define LOG_INFO 2 +#define LOG_WARNING 3 +#define LOG_ERROR 4 +#define LOG_DEBUG_STR "1" +#define LOG_INFO_STR "2" +#define LOG_WARNING_STR "3" +#define LOG_ERROR_STR "4" + +static int getLogLevel() +{ + int logLevel = LOG_INFO; + char* logLevelEnv = getenv("AISBENCH_LOG_LEVEL"); + if (logLevelEnv != nullptr) { + std::string logLevelStr = std::string(getenv("AISBENCH_LOG_LEVEL")); + if (logLevelStr == LOG_DEBUG_STR || logLevelStr == LOG_INFO_STR || + logLevelStr == LOG_WARNING_STR || logLevelStr == LOG_ERROR_STR) { + logLevel = std::stoi(logLevelStr); + } + } + return logLevel; +} + +static int FRIZY_LOG_LEVEL = getLogLevel(); + +#define PrintDbg_(mode, fmt, ...) PrintLog_debug(LogColorLightGreen_, mode, fmt, ##__VA_ARGS__) +#define PrintInfo_(mode, fmt, ...) PrintLog_(LogColorNone_, mode, fmt, ##__VA_ARGS__) +#define PrintWarn_(mode, fmt, ...) PrintLog_(LogColorLightYellow_, mode, fmt, ##__VA_ARGS__) +#define PrintErr_(mode, fmt, ...) PrintLog_(LogColorLightRed_, mode, fmt, ##__VA_ARGS__) + +#define LOG(level, fmt, ...) do { \ + if (level == LOG_DEBUG) { if (LOG_DEBUG >= FRIZY_LOG_LEVEL) {PrintDbg_("DEBUG", fmt, ##__VA_ARGS__);} \ + } else if (level == LOG_INFO) { if (LOG_INFO >= FRIZY_LOG_LEVEL) {PrintInfo_("INFO", fmt, ##__VA_ARGS__);} \ + } else if (level == LOG_WARNING) { if (LOG_WARNING >= FRIZY_LOG_LEVEL) {PrintWarn_("WARN", fmt, ##__VA_ARGS__);} \ + } else if (level == LOG_ERROR) { if (LOG_ERROR >= FRIZY_LOG_LEVEL) {PrintErr_("ERROR", fmt, ##__VA_ARGS__);} \ + } else { \ + } \ +} while (0) + + +#define SETLOGLEVEL(level) {FRIZY_LOG_LEVEL = level;} + +#define DEBUG(fmt, ...) LOG(LOG_DEBUG, fmt, ##__VA_ARGS__) +#define INFO(fmt, ...) LOG(LOG_INFO, fmt, ##__VA_ARGS__) +#define WARN(fmt, ...) LOG(LOG_WARNING, fmt, ##__VA_ARGS__) +#define ERROR(fmt, ...) LOG(LOG_ERROR, fmt, ##__VA_ARGS__) + + +#endif // LOG__H diff --git a/dependencies/transmit/src/tiny-AES-c/.gitignore b/dependencies/transmit/src/tiny-AES-c/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..8a9761d5f1d721605721f159df7e0aaaf3a0e087 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/.gitignore @@ -0,0 +1,4 @@ +/*.o +/test.elf +/test.map +test_package/build diff --git a/dependencies/transmit/src/tiny-AES-c/CMakeLists.txt b/dependencies/transmit/src/tiny-AES-c/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..5d637d5ed9f808450de5186238166278632810f6 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/CMakeLists.txt @@ -0,0 +1,8 @@ +SET(PROJECT_NAME tiny-aes) + +project(${PROJECT_NAME} C) + +set(CMAKE_C_FLAGS "-fPIC ${CMAKE_C_FLAGS}") + +set(LIBS_SRC aes.c) +ADD_LIBRARY(${PROJECT_NAME} STATIC ${LIBS_SRC}) \ No newline at end of file diff --git a/dependencies/transmit/src/tiny-AES-c/Makefile b/dependencies/transmit/src/tiny-AES-c/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..5581093437c3d7438953a9d6b009f9beffc924ec --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/Makefile @@ -0,0 +1,61 @@ +#CC = avr-gcc +#CFLAGS = -Wall -mmcu=atmega16 -Os -Wl,-Map,test.map +#OBJCOPY = avr-objcopy +CC = gcc +LD = gcc +AR = ar +ARFLAGS = rcs +CFLAGS = -Wall -Os -c +LDFLAGS = -Wall -Os -Wl,-Map,test.map +ifdef AES192 +CFLAGS += -DAES192=1 +endif +ifdef AES256 +CFLAGS += -DAES256=1 +endif + +OBJCOPYFLAGS = -j .text -O ihex +OBJCOPY = objcopy + +# include path to AVR library +INCLUDE_PATH = /usr/lib/avr/include +# splint static check +SPLINT = splint test.c aes.c -I$(INCLUDE_PATH) +charindex -unrecog + +default: test.elf + +.SILENT: +.PHONY: lint clean + +test.hex : test.elf + echo copy object-code to new image and format in hex + $(OBJCOPY) ${OBJCOPYFLAGS} $< $@ + +test.o : test.c aes.h aes.o + echo [CC] $@ $(CFLAGS) + $(CC) $(CFLAGS) -o $@ $< + +aes.o : aes.c aes.h + echo [CC] $@ $(CFLAGS) + $(CC) $(CFLAGS) -o $@ $< + +test.elf : aes.o test.o + echo [LD] $@ + $(LD) $(LDFLAGS) -o $@ $^ + +aes.a : aes.o + echo [AR] $@ + $(AR) $(ARFLAGS) $@ $^ + +lib : aes.a + +clean: + rm -f *.OBJ *.LST *.o *.gch *.out *.hex *.map *.elf *.a + +test: + make clean && make && ./test.elf + make clean && make AES192=1 && ./test.elf + make clean && make AES256=1 && ./test.elf + +lint: + $(call SPLINT) diff --git a/dependencies/transmit/src/tiny-AES-c/README.md b/dependencies/transmit/src/tiny-AES-c/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6811e0ecdc804a32fdd97caeb10198e0ee42e73e --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/README.md @@ -0,0 +1,83 @@ +![CI](https://github.com/kokke/tiny-AES-c/workflows/CI/badge.svg) +### Tiny AES in C + +This is a small and portable implementation of the AES [ECB](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_Codebook_.28ECB.29), [CTR](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_.28CTR.29) and [CBC](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Cipher_Block_Chaining_.28CBC.29) encryption algorithms written in C. + +You can override the default key-size of 128 bit with 192 or 256 bit by defining the symbols AES192 or AES256 in [`aes.h`](https://github.com/kokke/tiny-AES-c/blob/master/aes.h). + +The API is very simple and looks like this (I am using C99 ``-style annotated types): + +```C +/* Initialize context calling one of: */ +void AES_init_ctx(struct AES_ctx* ctx, const uint8_t* key); +void AES_init_ctx_iv(struct AES_ctx* ctx, const uint8_t* key, const uint8_t* iv); + +/* ... or reset IV at random point: */ +void AES_ctx_set_iv(struct AES_ctx* ctx, const uint8_t* iv); + +/* Then start encrypting and decrypting with the functions below: */ +void AES_ECB_encrypt(const struct AES_ctx* ctx, uint8_t* buf); +void AES_ECB_decrypt(const struct AES_ctx* ctx, uint8_t* buf); + +void AES_CBC_encrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length); +void AES_CBC_decrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length); + +/* Same function for encrypting as for decrypting in CTR mode */ +void AES_CTR_xcrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length); +``` + +Important notes: + * No padding is provided so for CBC and ECB all buffers should be multiples of 16 bytes. For padding [PKCS7](https://en.wikipedia.org/wiki/Padding_(cryptography)#PKCS7) is recommendable. + * ECB mode is considered unsafe for most uses and is not implemented in streaming mode. If you need this mode, call the function for every block of 16 bytes you need encrypted. See [wikipedia's article on ECB](https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Electronic_Codebook_(ECB)) for more details. + * This library is designed for small code size and simplicity, intended for cases where small binary size, low memory footprint and portability is more important than high performance. If speed is a concern, you can try more complex libraries, e.g. [Mbed TLS](https://tls.mbed.org/), [OpenSSL](https://www.openssl.org/) etc. + +You can choose to use any or all of the modes-of-operations, by defining the symbols CBC, CTR or ECB in [`aes.h`](https://github.com/kokke/tiny-AES-c/blob/master/aes.h) (read the comments for clarification). + +C++ users should `#include` [aes.hpp](https://github.com/kokke/tiny-AES-c/blob/master/aes.hpp) instead of [aes.h](https://github.com/kokke/tiny-AES-c/blob/master/aes.h) + +There is no built-in error checking or protection from out-of-bounds memory access errors as a result of malicious input. + +The module uses less than 200 bytes of RAM and 1-2K ROM when compiled for ARM, but YMMV depending on which modes are enabled. + +It is one of the smallest implementations in C I've seen yet, but do contact me if you know of something smaller (or have improvements to the code here). + +I've successfully used the code on 64bit x86, 32bit ARM and 8 bit AVR platforms. + + +GCC size output when only CTR mode is compiled for ARM: + + $ arm-none-eabi-gcc -Os -DCBC=0 -DECB=0 -DCTR=1 -c aes.c + $ size aes.o + text data bss dec hex filename + 1171 0 0 1171 493 aes.o + +.. and when compiling for the THUMB instruction set, we end up well below 1K in code size. + + $ arm-none-eabi-gcc -Os -mthumb -DCBC=0 -DECB=0 -DCTR=1 -c aes.c + $ size aes.o + text data bss dec hex filename + 903 0 0 903 387 aes.o + + +I am using the Free Software Foundation, ARM GCC compiler: + + $ arm-none-eabi-gcc --version + arm-none-eabi-gcc (4.8.4-1+11-1) 4.8.4 20141219 (release) + Copyright (C) 2013 Free Software Foundation, Inc. + This is free software; see the source for copying conditions. There is NO + warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + + + + +This implementation is verified against the data in: + +[National Institute of Standards and Technology Special Publication 800-38A 2001 ED](http://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38a.pdf) Appendix F: Example Vectors for Modes of Operation of the AES. + +The other appendices in the document are valuable for implementation details on e.g. padding, generation of IVs and nonces in CTR-mode etc. + + +A heartfelt thank-you to [all the nice people](https://github.com/kokke/tiny-AES-c/graphs/contributors) out there who have contributed to this project. + + +All material in this repository is in the public domain. diff --git a/dependencies/transmit/src/tiny-AES-c/aes.c b/dependencies/transmit/src/tiny-AES-c/aes.c new file mode 100644 index 0000000000000000000000000000000000000000..4481f7b24ec964019d38669842913fd571d28ba3 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/aes.c @@ -0,0 +1,572 @@ +/* + +This is an implementation of the AES algorithm, specifically ECB, CTR and CBC mode. +Block size can be chosen in aes.h - available choices are AES128, AES192, AES256. + +The implementation is verified against the test vectors in: + National Institute of Standards and Technology Special Publication 800-38A 2001 ED + +ECB-AES128 +---------- + + plain-text: + 6bc1bee22e409f96e93d7e117393172a + ae2d8a571e03ac9c9eb76fac45af8e51 + 30c81c46a35ce411e5fbc1191a0a52ef + f69f2445df4f9b17ad2b417be66c3710 + + key: + 2b7e151628aed2a6abf7158809cf4f3c + + resulting cipher + 3ad77bb40d7a3660a89ecaf32466ef97 + f5d3d58503b9699de785895a96fdbaaf + 43b1cd7f598ece23881b00e3ed030688 + 7b0c785e27e8ad3f8223207104725dd4 + + +NOTE: String length must be evenly divisible by 16byte (str_len % 16 == 0) + You should pad the end of the string with zeros if this is not the case. + For AES192/256 the key size is proportionally larger. + +*/ + + +/*****************************************************************************/ +/* Includes: */ +/*****************************************************************************/ +#include // CBC mode, for memset +#include "aes.h" + +/*****************************************************************************/ +/* Defines: */ +/*****************************************************************************/ +// The number of columns comprising a state in AES. This is a constant in AES. Value=4 +#define Nb 4 + +#if defined(AES256) && (AES256 == 1) + #define Nk 8 + #define Nr 14 +#elif defined(AES192) && (AES192 == 1) + #define Nk 6 + #define Nr 12 +#else + #define Nk 4 // The number of 32 bit words in a key. + #define Nr 10 // The number of rounds in AES Cipher. +#endif + +// jcallan@github points out that declaring Multiply as a function +// reduces code size considerably with the Keil ARM compiler. +// See this link for more information: https://github.com/kokke/tiny-AES-C/pull/3 +#ifndef MULTIPLY_AS_A_FUNCTION + #define MULTIPLY_AS_A_FUNCTION 0 +#endif + + + + +/*****************************************************************************/ +/* Private variables: */ +/*****************************************************************************/ +// state - array holding the intermediate results during decryption. +typedef uint8_t state_t[4][4]; + + + +// The lookup-tables are marked const so they can be placed in read-only storage instead of RAM +// The numbers below can be computed dynamically trading ROM for RAM - +// This can be useful in (embedded) bootloader applications, where ROM is often limited. +static const uint8_t sbox[256] = { + //0 1 2 3 4 5 6 7 8 9 A B C D E F + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, + 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, + 0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, + 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, + 0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, + 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, + 0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, + 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, + 0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, + 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, + 0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16 }; + +#if (defined(CBC) && CBC == 1) || (defined(ECB) && ECB == 1) +static const uint8_t rsbox[256] = { + 0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d }; +#endif + +// The round constant word array, Rcon[i], contains the values given by +// x to the power (i-1) being powers of x (x is denoted as {02}) in the field GF(2^8) +static const uint8_t Rcon[11] = { + 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36 }; + +/* + * Jordan Goulder points out in PR #12 (https://github.com/kokke/tiny-AES-C/pull/12), + * that you can remove most of the elements in the Rcon array, because they are unused. + * + * From Wikipedia's article on the Rijndael key schedule @ https://en.wikipedia.org/wiki/Rijndael_key_schedule#Rcon + * + * "Only the first some of these constants are actually used – up to rcon[10] for AES-128 (as 11 round keys are needed), + * up to rcon[8] for AES-192, up to rcon[7] for AES-256. rcon[0] is not used in AES algorithm." + */ + + +/*****************************************************************************/ +/* Private functions: */ +/*****************************************************************************/ +/* +static uint8_t getSBoxValue(uint8_t num) +{ + return sbox[num]; +} +*/ +#define getSBoxValue(num) (sbox[(num)]) + +// This function produces Nb(Nr+1) round keys. The round keys are used in each round to decrypt the states. +static void KeyExpansion(uint8_t* RoundKey, const uint8_t* Key) +{ + unsigned i, j, k; + uint8_t tempa[4]; // Used for the column/row operations + + // The first round key is the key itself. + for (i = 0; i < Nk; ++i) + { + RoundKey[(i * 4) + 0] = Key[(i * 4) + 0]; + RoundKey[(i * 4) + 1] = Key[(i * 4) + 1]; + RoundKey[(i * 4) + 2] = Key[(i * 4) + 2]; + RoundKey[(i * 4) + 3] = Key[(i * 4) + 3]; + } + + // All other round keys are found from the previous round keys. + for (i = Nk; i < Nb * (Nr + 1); ++i) + { + { + k = (i - 1) * 4; + tempa[0]=RoundKey[k + 0]; + tempa[1]=RoundKey[k + 1]; + tempa[2]=RoundKey[k + 2]; + tempa[3]=RoundKey[k + 3]; + + } + + if (i % Nk == 0) + { + // This function shifts the 4 bytes in a word to the left once. + // [a0,a1,a2,a3] becomes [a1,a2,a3,a0] + + // Function RotWord() + { + const uint8_t u8tmp = tempa[0]; + tempa[0] = tempa[1]; + tempa[1] = tempa[2]; + tempa[2] = tempa[3]; + tempa[3] = u8tmp; + } + + // SubWord() is a function that takes a four-byte input word and + // applies the S-box to each of the four bytes to produce an output word. + + // Function Subword() + { + tempa[0] = getSBoxValue(tempa[0]); + tempa[1] = getSBoxValue(tempa[1]); + tempa[2] = getSBoxValue(tempa[2]); + tempa[3] = getSBoxValue(tempa[3]); + } + + tempa[0] = tempa[0] ^ Rcon[i/Nk]; + } +#if defined(AES256) && (AES256 == 1) + if (i % Nk == 4) + { + // Function Subword() + { + tempa[0] = getSBoxValue(tempa[0]); + tempa[1] = getSBoxValue(tempa[1]); + tempa[2] = getSBoxValue(tempa[2]); + tempa[3] = getSBoxValue(tempa[3]); + } + } +#endif + j = i * 4; k=(i - Nk) * 4; + RoundKey[j + 0] = RoundKey[k + 0] ^ tempa[0]; + RoundKey[j + 1] = RoundKey[k + 1] ^ tempa[1]; + RoundKey[j + 2] = RoundKey[k + 2] ^ tempa[2]; + RoundKey[j + 3] = RoundKey[k + 3] ^ tempa[3]; + } +} + +void AES_init_ctx(struct AES_ctx* ctx, const uint8_t* key) +{ + KeyExpansion(ctx->RoundKey, key); +} +#if (defined(CBC) && (CBC == 1)) || (defined(CTR) && (CTR == 1)) +void AES_init_ctx_iv(struct AES_ctx* ctx, const uint8_t* key, const uint8_t* iv) +{ + KeyExpansion(ctx->RoundKey, key); + memcpy (ctx->Iv, iv, AES_BLOCKLEN); +} +void AES_ctx_set_iv(struct AES_ctx* ctx, const uint8_t* iv) +{ + memcpy (ctx->Iv, iv, AES_BLOCKLEN); +} +#endif + +// This function adds the round key to state. +// The round key is added to the state by an XOR function. +static void AddRoundKey(uint8_t round, state_t* state, const uint8_t* RoundKey) +{ + uint8_t i,j; + for (i = 0; i < 4; ++i) + { + for (j = 0; j < 4; ++j) + { + (*state)[i][j] ^= RoundKey[(round * Nb * 4) + (i * Nb) + j]; + } + } +} + +// The SubBytes Function Substitutes the values in the +// state matrix with values in an S-box. +static void SubBytes(state_t* state) +{ + uint8_t i, j; + for (i = 0; i < 4; ++i) + { + for (j = 0; j < 4; ++j) + { + (*state)[j][i] = getSBoxValue((*state)[j][i]); + } + } +} + +// The ShiftRows() function shifts the rows in the state to the left. +// Each row is shifted with different offset. +// Offset = Row number. So the first row is not shifted. +static void ShiftRows(state_t* state) +{ + uint8_t temp; + + // Rotate first row 1 columns to left + temp = (*state)[0][1]; + (*state)[0][1] = (*state)[1][1]; + (*state)[1][1] = (*state)[2][1]; + (*state)[2][1] = (*state)[3][1]; + (*state)[3][1] = temp; + + // Rotate second row 2 columns to left + temp = (*state)[0][2]; + (*state)[0][2] = (*state)[2][2]; + (*state)[2][2] = temp; + + temp = (*state)[1][2]; + (*state)[1][2] = (*state)[3][2]; + (*state)[3][2] = temp; + + // Rotate third row 3 columns to left + temp = (*state)[0][3]; + (*state)[0][3] = (*state)[3][3]; + (*state)[3][3] = (*state)[2][3]; + (*state)[2][3] = (*state)[1][3]; + (*state)[1][3] = temp; +} + +static uint8_t xtime(uint8_t x) +{ + return ((x<<1) ^ (((x>>7) & 1) * 0x1b)); +} + +// MixColumns function mixes the columns of the state matrix +static void MixColumns(state_t* state) +{ + uint8_t i; + uint8_t Tmp, Tm, t; + for (i = 0; i < 4; ++i) + { + t = (*state)[i][0]; + Tmp = (*state)[i][0] ^ (*state)[i][1] ^ (*state)[i][2] ^ (*state)[i][3] ; + Tm = (*state)[i][0] ^ (*state)[i][1] ; Tm = xtime(Tm); (*state)[i][0] ^= Tm ^ Tmp ; + Tm = (*state)[i][1] ^ (*state)[i][2] ; Tm = xtime(Tm); (*state)[i][1] ^= Tm ^ Tmp ; + Tm = (*state)[i][2] ^ (*state)[i][3] ; Tm = xtime(Tm); (*state)[i][2] ^= Tm ^ Tmp ; + Tm = (*state)[i][3] ^ t ; Tm = xtime(Tm); (*state)[i][3] ^= Tm ^ Tmp ; + } +} + +// Multiply is used to multiply numbers in the field GF(2^8) +// Note: The last call to xtime() is unneeded, but often ends up generating a smaller binary +// The compiler seems to be able to vectorize the operation better this way. +// See https://github.com/kokke/tiny-AES-c/pull/34 +#if MULTIPLY_AS_A_FUNCTION +static uint8_t Multiply(uint8_t x, uint8_t y) +{ + return (((y & 1) * x) ^ + ((y>>1 & 1) * xtime(x)) ^ + ((y>>2 & 1) * xtime(xtime(x))) ^ + ((y>>3 & 1) * xtime(xtime(xtime(x)))) ^ + ((y>>4 & 1) * xtime(xtime(xtime(xtime(x)))))); /* this last call to xtime() can be omitted */ + } +#else +#define Multiply(x, y) \ + ( ((y & 1) * x) ^ \ + ((y>>1 & 1) * xtime(x)) ^ \ + ((y>>2 & 1) * xtime(xtime(x))) ^ \ + ((y>>3 & 1) * xtime(xtime(xtime(x)))) ^ \ + ((y>>4 & 1) * xtime(xtime(xtime(xtime(x)))))) \ + +#endif + +#if (defined(CBC) && CBC == 1) || (defined(ECB) && ECB == 1) +/* +static uint8_t getSBoxInvert(uint8_t num) +{ + return rsbox[num]; +} +*/ +#define getSBoxInvert(num) (rsbox[(num)]) + +// MixColumns function mixes the columns of the state matrix. +// The method used to multiply may be difficult to understand for the inexperienced. +// Please use the references to gain more information. +static void InvMixColumns(state_t* state) +{ + int i; + uint8_t a, b, c, d; + for (i = 0; i < 4; ++i) + { + a = (*state)[i][0]; + b = (*state)[i][1]; + c = (*state)[i][2]; + d = (*state)[i][3]; + + (*state)[i][0] = Multiply(a, 0x0e) ^ Multiply(b, 0x0b) ^ Multiply(c, 0x0d) ^ Multiply(d, 0x09); + (*state)[i][1] = Multiply(a, 0x09) ^ Multiply(b, 0x0e) ^ Multiply(c, 0x0b) ^ Multiply(d, 0x0d); + (*state)[i][2] = Multiply(a, 0x0d) ^ Multiply(b, 0x09) ^ Multiply(c, 0x0e) ^ Multiply(d, 0x0b); + (*state)[i][3] = Multiply(a, 0x0b) ^ Multiply(b, 0x0d) ^ Multiply(c, 0x09) ^ Multiply(d, 0x0e); + } +} + + +// The SubBytes Function Substitutes the values in the +// state matrix with values in an S-box. +static void InvSubBytes(state_t* state) +{ + uint8_t i, j; + for (i = 0; i < 4; ++i) + { + for (j = 0; j < 4; ++j) + { + (*state)[j][i] = getSBoxInvert((*state)[j][i]); + } + } +} + +static void InvShiftRows(state_t* state) +{ + uint8_t temp; + + // Rotate first row 1 columns to right + temp = (*state)[3][1]; + (*state)[3][1] = (*state)[2][1]; + (*state)[2][1] = (*state)[1][1]; + (*state)[1][1] = (*state)[0][1]; + (*state)[0][1] = temp; + + // Rotate second row 2 columns to right + temp = (*state)[0][2]; + (*state)[0][2] = (*state)[2][2]; + (*state)[2][2] = temp; + + temp = (*state)[1][2]; + (*state)[1][2] = (*state)[3][2]; + (*state)[3][2] = temp; + + // Rotate third row 3 columns to right + temp = (*state)[0][3]; + (*state)[0][3] = (*state)[1][3]; + (*state)[1][3] = (*state)[2][3]; + (*state)[2][3] = (*state)[3][3]; + (*state)[3][3] = temp; +} +#endif // #if (defined(CBC) && CBC == 1) || (defined(ECB) && ECB == 1) + +// Cipher is the main function that encrypts the PlainText. +static void Cipher(state_t* state, const uint8_t* RoundKey) +{ + uint8_t round = 0; + + // Add the First round key to the state before starting the rounds. + AddRoundKey(0, state, RoundKey); + + // There will be Nr rounds. + // The first Nr-1 rounds are identical. + // These Nr rounds are executed in the loop below. + // Last one without MixColumns() + for (round = 1; ; ++round) + { + SubBytes(state); + ShiftRows(state); + if (round == Nr) { + break; + } + MixColumns(state); + AddRoundKey(round, state, RoundKey); + } + // Add round key to last round + AddRoundKey(Nr, state, RoundKey); +} + +#if (defined(CBC) && CBC == 1) || (defined(ECB) && ECB == 1) +static void InvCipher(state_t* state, const uint8_t* RoundKey) +{ + uint8_t round = 0; + + // Add the First round key to the state before starting the rounds. + AddRoundKey(Nr, state, RoundKey); + + // There will be Nr rounds. + // The first Nr-1 rounds are identical. + // These Nr rounds are executed in the loop below. + // Last one without InvMixColumn() + for (round = (Nr - 1); ; --round) + { + InvShiftRows(state); + InvSubBytes(state); + AddRoundKey(round, state, RoundKey); + if (round == 0) { + break; + } + InvMixColumns(state); + } + +} +#endif // #if (defined(CBC) && CBC == 1) || (defined(ECB) && ECB == 1) + +/*****************************************************************************/ +/* Public functions: */ +/*****************************************************************************/ +#if defined(ECB) && (ECB == 1) + + +void AES_ECB_encrypt(const struct AES_ctx* ctx, uint8_t* buf) +{ + // The next function call encrypts the PlainText with the Key using AES algorithm. + Cipher((state_t*)buf, ctx->RoundKey); +} + +void AES_ECB_decrypt(const struct AES_ctx* ctx, uint8_t* buf) +{ + // The next function call decrypts the PlainText with the Key using AES algorithm. + InvCipher((state_t*)buf, ctx->RoundKey); +} + + +#endif // #if defined(ECB) && (ECB == 1) + + + + + +#if defined(CBC) && (CBC == 1) + + +static void XorWithIv(uint8_t* buf, const uint8_t* Iv) +{ + uint8_t i; + for (i = 0; i < AES_BLOCKLEN; ++i) // The block in AES is always 128bit no matter the key size + { + buf[i] ^= Iv[i]; + } +} + +void AES_CBC_encrypt_buffer(struct AES_ctx *ctx, uint8_t* buf, size_t length) +{ + size_t i; + uint8_t *Iv = ctx->Iv; + for (i = 0; i < length; i += AES_BLOCKLEN) + { + XorWithIv(buf, Iv); + Cipher((state_t*)buf, ctx->RoundKey); + Iv = buf; + buf += AES_BLOCKLEN; + } + /* store Iv in ctx for next call */ + memcpy(ctx->Iv, Iv, AES_BLOCKLEN); +} + +void AES_CBC_decrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length) +{ + size_t i; + uint8_t storeNextIv[AES_BLOCKLEN]; + for (i = 0; i < length; i += AES_BLOCKLEN) + { + memcpy(storeNextIv, buf, AES_BLOCKLEN); + InvCipher((state_t*)buf, ctx->RoundKey); + XorWithIv(buf, ctx->Iv); + memcpy(ctx->Iv, storeNextIv, AES_BLOCKLEN); + buf += AES_BLOCKLEN; + } + +} + +#endif // #if defined(CBC) && (CBC == 1) + + + +#if defined(CTR) && (CTR == 1) + +/* Symmetrical operation: same function for encrypting as for decrypting. Note any IV/nonce should never be reused with the same key */ +void AES_CTR_xcrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length) +{ + uint8_t buffer[AES_BLOCKLEN]; + + size_t i; + int bi; + for (i = 0, bi = AES_BLOCKLEN; i < length; ++i, ++bi) + { + if (bi == AES_BLOCKLEN) /* we need to regen xor compliment in buffer */ + { + + memcpy(buffer, ctx->Iv, AES_BLOCKLEN); + Cipher((state_t*)buffer,ctx->RoundKey); + + /* Increment Iv and handle overflow */ + for (bi = (AES_BLOCKLEN - 1); bi >= 0; --bi) + { + /* inc will overflow */ + if (ctx->Iv[bi] == 255) + { + ctx->Iv[bi] = 0; + continue; + } + ctx->Iv[bi] += 1; + break; + } + bi = 0; + } + + buf[i] = (buf[i] ^ buffer[bi]); + } +} + +#endif // #if defined(CTR) && (CTR == 1) + diff --git a/dependencies/transmit/src/tiny-AES-c/aes.h b/dependencies/transmit/src/tiny-AES-c/aes.h new file mode 100644 index 0000000000000000000000000000000000000000..d83170eb693331e213eb901178171f26d3f24798 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/aes.h @@ -0,0 +1,90 @@ +#ifndef _AES_H_ +#define _AES_H_ + +#include +#include +// #define the macros below to 1/0 to enable/disable the mode of operation. +// +// CBC enables AES encryption in CBC-mode of operation. +// CTR enables encryption in counter-mode. +// ECB enables the basic ECB 16-byte block algorithm. All can be enabled simultaneously. + +// The #ifndef-guard allows it to be configured before #include'ing or at compile time. +#ifndef CBC + #define CBC 1 +#endif + +#ifndef ECB + #define ECB 1 +#endif + +#ifndef CTR + #define CTR 1 +#endif + + +//#define AES128 1 +//#define AES192 1 +#define AES256 1 + +#define AES_BLOCKLEN 16 // Block length in bytes - AES is 128b block only + +#if defined(AES256) && (AES256 == 1) + #define AES_KEYLEN 32 + #define AES_keyExpSize 240 +#elif defined(AES192) && (AES192 == 1) + #define AES_KEYLEN 24 + #define AES_keyExpSize 208 +#else + #define AES_KEYLEN 16 // Key length in bytes + #define AES_keyExpSize 176 +#endif + +struct AES_ctx +{ + uint8_t RoundKey[AES_keyExpSize]; +#if (defined(CBC) && (CBC == 1)) || (defined(CTR) && (CTR == 1)) + uint8_t Iv[AES_BLOCKLEN]; +#endif +}; + +extern void AES_init_ctx(struct AES_ctx* ctx, const uint8_t* key); +#if (defined(CBC) && (CBC == 1)) || (defined(CTR) && (CTR == 1)) +extern void AES_init_ctx_iv(struct AES_ctx* ctx, const uint8_t* key, const uint8_t* iv); +extern void AES_ctx_set_iv(struct AES_ctx* ctx, const uint8_t* iv); +#endif + +#if defined(ECB) && (ECB == 1) +// buffer size is exactly AES_BLOCKLEN bytes; +// you need only AES_init_ctx as IV is not used in ECB +// NB: ECB is considered insecure for most uses +extern void AES_ECB_encrypt(const struct AES_ctx* ctx, uint8_t* buf); +extern void AES_ECB_decrypt(const struct AES_ctx* ctx, uint8_t* buf); + +#endif // #if defined(ECB) && (ECB == !) + + +#if defined(CBC) && (CBC == 1) +// buffer size MUST be mutile of AES_BLOCKLEN; +// Suggest https://en.wikipedia.org/wiki/Padding_(cryptography)#PKCS7 for padding scheme +// NOTES: you need to set IV in ctx via AES_init_ctx_iv() or AES_ctx_set_iv() +// no IV should ever be reused with the same key +extern void AES_CBC_encrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length); +extern void AES_CBC_decrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length); + +#endif // #if defined(CBC) && (CBC == 1) + + +#if defined(CTR) && (CTR == 1) + +// Same function for encrypting as for decrypting. +// IV is incremented for every block, and used after encryption as XOR-compliment for output +// Suggesting https://en.wikipedia.org/wiki/Padding_(cryptography)#PKCS7 for padding scheme +// NOTES: you need to set IV in ctx with AES_init_ctx_iv() or AES_ctx_set_iv() +// no IV should ever be reused with the same key +extern void AES_CTR_xcrypt_buffer(struct AES_ctx* ctx, uint8_t* buf, size_t length); + +#endif // #if defined(CTR) && (CTR == 1) + + +#endif // _AES_H_ diff --git a/dependencies/transmit/src/tiny-AES-c/aes.hpp b/dependencies/transmit/src/tiny-AES-c/aes.hpp new file mode 100644 index 0000000000000000000000000000000000000000..ade16425dedf43173d66f38951b2d52e43226df3 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/aes.hpp @@ -0,0 +1,12 @@ +#ifndef _AES_HPP_ +#define _AES_HPP_ + +#ifndef __cplusplus +#error Do not include the hpp header in a c project! +#endif //__cplusplus + +extern "C" { +#include "aes.h" +} + +#endif //_AES_HPP_ diff --git a/dependencies/transmit/src/tiny-AES-c/conanfile.py b/dependencies/transmit/src/tiny-AES-c/conanfile.py new file mode 100644 index 0000000000000000000000000000000000000000..8a75744eaf5b25f87f062b543862bee8560b2b22 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/conanfile.py @@ -0,0 +1,73 @@ +from conans import ConanFile, CMake +from conans.errors import ConanException + + +class TinyAesCConan(ConanFile): + name = "tiny-AES-c" + version = "1.0.0" + license = "The Unlicense" + url = "https://github.com/kokke/tiny-AES-c" + description = "Small portable AES128/192/256 in C" + topics = ("encryption", "crypto", "AES") + settings = "os", "compiler", "build_type", "arch" + + generators = "cmake" + exports_sources = ["CMakeLists.txt", "*.c", '*.h', '*.hpp'] + exports = ["unlicense.txt"] + + _options_dict = { + # enable AES128 + "AES128": [True, False], + + # enable AES192 + "AES192": [True, False], + + # enable AES256 + "AES256": [True, False], + + # enable AES encryption in CBC-mode of operation + "CBC": [True, False], + + # enable the basic ECB 16-byte block algorithm + "ECB": [True, False], + + # enable encryption in counter-mode + "CTR": [True, False], + } + + options = _options_dict + + default_options = { + "AES128": True, + "AES192": False, + "AES256": False, + "CBC": True, + "ECB": True, + "CTR": True + } + + def configure(self): + if not self.options.CBC and not self.options.ECB and not self.options.CTR: + raise ConanException("Need to at least specify one of CBC, ECB or CTR modes") + + if not self.options.AES128 and not self.options.AES192 and not self.options.AES256: + raise ConanException("Need to at least specify one of AES{128, 192, 256} modes") + + def build(self): + cmake = CMake(self) + + for key in self._options_dict.keys(): + if self.options[key]: + cmake.definitions["CMAKE_CFLAGS"].append(key) + + cmake.configure() + cmake.build() + + def package(self): + self.copy("*.h", dst="include") + self.copy("*.hpp", dst="include") + self.copy("*.a", dst="lib", keep_path=False) + self.copy("unlicense.txt") + + def package_info(self): + self.cpp_info.libs = ["tiny-aes"] diff --git a/dependencies/transmit/src/tiny-AES-c/library.json b/dependencies/transmit/src/tiny-AES-c/library.json new file mode 100644 index 0000000000000000000000000000000000000000..d7abe89a58c463bef83b7552ffa45729f083fd8c --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/library.json @@ -0,0 +1,13 @@ +{ + "name": "tiny-AES-c", + "keywords": "cryptography, aes", + "description": "Small portable AES128/192/256 in C", + "repository": + { + "type": "git", + "url": "https://github.com/kokke/tiny-AES-c.git" + }, + "frameworks": "*", + "platforms": "*", + "examples": "test.c" +} diff --git a/dependencies/transmit/src/tiny-AES-c/library.properties b/dependencies/transmit/src/tiny-AES-c/library.properties new file mode 100644 index 0000000000000000000000000000000000000000..39082cd031978b17b5763d145c3abbafc9b1d5c6 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/library.properties @@ -0,0 +1,11 @@ +name=tiny-AES-c +version=1.0.0 +author=kokke +maintainer=kokke +sentence=A small and portable implementation of the AES ECB, CTR and CBC encryption algorithms written in C. +paragraph=You can override the default key-size of 128 bit with 192 or 256 bit by defining the symbols AES192 or AES256 in aes.h. The API is very simple. No padding is provided so for CBC and ECB all buffers should be multiples of 16 bytes. +category=Data Processing +url=https://github.com/kokke/tiny-AES-c +repository=https://github.com/kokke/tiny-AES-c.git +architectures=* +includes=aes.h diff --git a/dependencies/transmit/src/tiny-AES-c/test.c b/dependencies/transmit/src/tiny-AES-c/test.c new file mode 100644 index 0000000000000000000000000000000000000000..679628310c5017fb9367af6c02f796fe49b36af4 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/test.c @@ -0,0 +1,316 @@ +#include +#include +#include + +// Enable ECB, CTR and CBC mode. Note this can be done before including aes.h or at compile-time. +// E.g. with GCC by using the -D flag: gcc -c aes.c -DCBC=0 -DCTR=1 -DECB=1 +#define CBC 1 +#define CTR 1 +#define ECB 1 + +#include "aes.h" + + +static void phex(uint8_t* str); +static int test_encrypt_cbc(void); +static int test_decrypt_cbc(void); +static int test_encrypt_ctr(void); +static int test_decrypt_ctr(void); +static int test_encrypt_ecb(void); +static int test_decrypt_ecb(void); +static void test_encrypt_ecb_verbose(void); + + +int main(void) +{ + int exit; + +#if defined(AES256) + printf("\nTesting AES256\n\n"); +#elif defined(AES192) + printf("\nTesting AES192\n\n"); +#elif defined(AES128) + printf("\nTesting AES128\n\n"); +#else + printf("You need to specify a symbol between AES128, AES192 or AES256. Exiting"); + return 0; +#endif + + exit = test_encrypt_cbc() + test_decrypt_cbc() + + test_encrypt_ctr() + test_decrypt_ctr() + + test_decrypt_ecb() + test_encrypt_ecb(); + test_encrypt_ecb_verbose(); + + return exit; +} + + +// prints string as hex +static void phex(uint8_t* str) +{ + +#if defined(AES256) + uint8_t len = 32; +#elif defined(AES192) + uint8_t len = 24; +#elif defined(AES128) + uint8_t len = 16; +#endif + + unsigned char i; + for (i = 0; i < len; ++i) + printf("%.2x", str[i]); + printf("\n"); +} + +static void test_encrypt_ecb_verbose(void) +{ + // Example of more verbose verification + + uint8_t i; + + // 128bit key + uint8_t key[16] = { (uint8_t) 0x2b, (uint8_t) 0x7e, (uint8_t) 0x15, (uint8_t) 0x16, (uint8_t) 0x28, (uint8_t) 0xae, (uint8_t) 0xd2, (uint8_t) 0xa6, (uint8_t) 0xab, (uint8_t) 0xf7, (uint8_t) 0x15, (uint8_t) 0x88, (uint8_t) 0x09, (uint8_t) 0xcf, (uint8_t) 0x4f, (uint8_t) 0x3c }; + // 512bit text + uint8_t plain_text[64] = { (uint8_t) 0x6b, (uint8_t) 0xc1, (uint8_t) 0xbe, (uint8_t) 0xe2, (uint8_t) 0x2e, (uint8_t) 0x40, (uint8_t) 0x9f, (uint8_t) 0x96, (uint8_t) 0xe9, (uint8_t) 0x3d, (uint8_t) 0x7e, (uint8_t) 0x11, (uint8_t) 0x73, (uint8_t) 0x93, (uint8_t) 0x17, (uint8_t) 0x2a, + (uint8_t) 0xae, (uint8_t) 0x2d, (uint8_t) 0x8a, (uint8_t) 0x57, (uint8_t) 0x1e, (uint8_t) 0x03, (uint8_t) 0xac, (uint8_t) 0x9c, (uint8_t) 0x9e, (uint8_t) 0xb7, (uint8_t) 0x6f, (uint8_t) 0xac, (uint8_t) 0x45, (uint8_t) 0xaf, (uint8_t) 0x8e, (uint8_t) 0x51, + (uint8_t) 0x30, (uint8_t) 0xc8, (uint8_t) 0x1c, (uint8_t) 0x46, (uint8_t) 0xa3, (uint8_t) 0x5c, (uint8_t) 0xe4, (uint8_t) 0x11, (uint8_t) 0xe5, (uint8_t) 0xfb, (uint8_t) 0xc1, (uint8_t) 0x19, (uint8_t) 0x1a, (uint8_t) 0x0a, (uint8_t) 0x52, (uint8_t) 0xef, + (uint8_t) 0xf6, (uint8_t) 0x9f, (uint8_t) 0x24, (uint8_t) 0x45, (uint8_t) 0xdf, (uint8_t) 0x4f, (uint8_t) 0x9b, (uint8_t) 0x17, (uint8_t) 0xad, (uint8_t) 0x2b, (uint8_t) 0x41, (uint8_t) 0x7b, (uint8_t) 0xe6, (uint8_t) 0x6c, (uint8_t) 0x37, (uint8_t) 0x10 }; + + // print text to encrypt, key and IV + printf("ECB encrypt verbose:\n\n"); + printf("plain text:\n"); + for (i = (uint8_t) 0; i < (uint8_t) 4; ++i) + { + phex(plain_text + i * (uint8_t) 16); + } + printf("\n"); + + printf("key:\n"); + phex(key); + printf("\n"); + + // print the resulting cipher as 4 x 16 byte strings + printf("ciphertext:\n"); + + struct AES_ctx ctx; + AES_init_ctx(&ctx, key); + + for (i = 0; i < 4; ++i) + { + AES_ECB_encrypt(&ctx, plain_text + (i * 16)); + phex(plain_text + (i * 16)); + } + printf("\n"); +} + + +static int test_encrypt_ecb(void) +{ +#if defined(AES256) + uint8_t key[] = { 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 }; + uint8_t out[] = { 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c, 0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8 }; +#elif defined(AES192) + uint8_t key[] = { 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b }; + uint8_t out[] = { 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f, 0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc }; +#elif defined(AES128) + uint8_t key[] = { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; + uint8_t out[] = { 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60, 0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97 }; +#endif + + uint8_t in[] = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a }; + struct AES_ctx ctx; + + AES_init_ctx(&ctx, key); + AES_ECB_encrypt(&ctx, in); + + printf("ECB encrypt: "); + + if (0 == memcmp((char*) out, (char*) in, 16)) { + printf("SUCCESS!\n"); + return(0); + } else { + printf("FAILURE!\n"); + return(1); + } +} + +static int test_decrypt_cbc(void) +{ + +#if defined(AES256) + uint8_t key[] = { 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 }; + uint8_t in[] = { 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba, 0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6, + 0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d, 0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d, + 0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf, 0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61, + 0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc, 0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b }; +#elif defined(AES192) + uint8_t key[] = { 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b }; + uint8_t in[] = { 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d, 0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8, + 0xb4, 0xd9, 0xad, 0xa9, 0xad, 0x7d, 0xed, 0xf4, 0xe5, 0xe7, 0x38, 0x76, 0x3f, 0x69, 0x14, 0x5a, + 0x57, 0x1b, 0x24, 0x20, 0x12, 0xfb, 0x7a, 0xe0, 0x7f, 0xa9, 0xba, 0xac, 0x3d, 0xf1, 0x02, 0xe0, + 0x08, 0xb0, 0xe2, 0x79, 0x88, 0x59, 0x88, 0x81, 0xd9, 0x20, 0xa9, 0xe6, 0x4f, 0x56, 0x15, 0xcd }; +#elif defined(AES128) + uint8_t key[] = { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; + uint8_t in[] = { 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d, + 0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2, + 0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b, 0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16, + 0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09, 0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7 }; +#endif + uint8_t iv[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; + uint8_t out[] = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 }; +// uint8_t buffer[64]; + struct AES_ctx ctx; + + AES_init_ctx_iv(&ctx, key, iv); + AES_CBC_decrypt_buffer(&ctx, in, 64); + + printf("CBC decrypt: "); + + if (0 == memcmp((char*) out, (char*) in, 64)) { + printf("SUCCESS!\n"); + return(0); + } else { + printf("FAILURE!\n"); + return(1); + } +} + +static int test_encrypt_cbc(void) +{ +#if defined(AES256) + uint8_t key[] = { 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 }; + uint8_t out[] = { 0xf5, 0x8c, 0x4c, 0x04, 0xd6, 0xe5, 0xf1, 0xba, 0x77, 0x9e, 0xab, 0xfb, 0x5f, 0x7b, 0xfb, 0xd6, + 0x9c, 0xfc, 0x4e, 0x96, 0x7e, 0xdb, 0x80, 0x8d, 0x67, 0x9f, 0x77, 0x7b, 0xc6, 0x70, 0x2c, 0x7d, + 0x39, 0xf2, 0x33, 0x69, 0xa9, 0xd9, 0xba, 0xcf, 0xa5, 0x30, 0xe2, 0x63, 0x04, 0x23, 0x14, 0x61, + 0xb2, 0xeb, 0x05, 0xe2, 0xc3, 0x9b, 0xe9, 0xfc, 0xda, 0x6c, 0x19, 0x07, 0x8c, 0x6a, 0x9d, 0x1b }; +#elif defined(AES192) + uint8_t key[] = { 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b }; + uint8_t out[] = { 0x4f, 0x02, 0x1d, 0xb2, 0x43, 0xbc, 0x63, 0x3d, 0x71, 0x78, 0x18, 0x3a, 0x9f, 0xa0, 0x71, 0xe8, + 0xb4, 0xd9, 0xad, 0xa9, 0xad, 0x7d, 0xed, 0xf4, 0xe5, 0xe7, 0x38, 0x76, 0x3f, 0x69, 0x14, 0x5a, + 0x57, 0x1b, 0x24, 0x20, 0x12, 0xfb, 0x7a, 0xe0, 0x7f, 0xa9, 0xba, 0xac, 0x3d, 0xf1, 0x02, 0xe0, + 0x08, 0xb0, 0xe2, 0x79, 0x88, 0x59, 0x88, 0x81, 0xd9, 0x20, 0xa9, 0xe6, 0x4f, 0x56, 0x15, 0xcd }; +#elif defined(AES128) + uint8_t key[] = { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; + uint8_t out[] = { 0x76, 0x49, 0xab, 0xac, 0x81, 0x19, 0xb2, 0x46, 0xce, 0xe9, 0x8e, 0x9b, 0x12, 0xe9, 0x19, 0x7d, + 0x50, 0x86, 0xcb, 0x9b, 0x50, 0x72, 0x19, 0xee, 0x95, 0xdb, 0x11, 0x3a, 0x91, 0x76, 0x78, 0xb2, + 0x73, 0xbe, 0xd6, 0xb8, 0xe3, 0xc1, 0x74, 0x3b, 0x71, 0x16, 0xe6, 0x9e, 0x22, 0x22, 0x95, 0x16, + 0x3f, 0xf1, 0xca, 0xa1, 0x68, 0x1f, 0xac, 0x09, 0x12, 0x0e, 0xca, 0x30, 0x75, 0x86, 0xe1, 0xa7 }; +#endif + uint8_t iv[] = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }; + uint8_t in[] = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 }; + struct AES_ctx ctx; + + AES_init_ctx_iv(&ctx, key, iv); + AES_CBC_encrypt_buffer(&ctx, in, 64); + + printf("CBC encrypt: "); + + if (0 == memcmp((char*) out, (char*) in, 64)) { + printf("SUCCESS!\n"); + return(0); + } else { + printf("FAILURE!\n"); + return(1); + } +} + +static int test_xcrypt_ctr(const char* xcrypt); +static int test_encrypt_ctr(void) +{ + return test_xcrypt_ctr("encrypt"); +} + +static int test_decrypt_ctr(void) +{ + return test_xcrypt_ctr("decrypt"); +} + +static int test_xcrypt_ctr(const char* xcrypt) +{ +#if defined(AES256) + uint8_t key[32] = { 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 }; + uint8_t in[64] = { 0x60, 0x1e, 0xc3, 0x13, 0x77, 0x57, 0x89, 0xa5, 0xb7, 0xa7, 0xf5, 0x04, 0xbb, 0xf3, 0xd2, 0x28, + 0xf4, 0x43, 0xe3, 0xca, 0x4d, 0x62, 0xb5, 0x9a, 0xca, 0x84, 0xe9, 0x90, 0xca, 0xca, 0xf5, 0xc5, + 0x2b, 0x09, 0x30, 0xda, 0xa2, 0x3d, 0xe9, 0x4c, 0xe8, 0x70, 0x17, 0xba, 0x2d, 0x84, 0x98, 0x8d, + 0xdf, 0xc9, 0xc5, 0x8d, 0xb6, 0x7a, 0xad, 0xa6, 0x13, 0xc2, 0xdd, 0x08, 0x45, 0x79, 0x41, 0xa6 }; +#elif defined(AES192) + uint8_t key[24] = { 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b }; + uint8_t in[64] = { 0x1a, 0xbc, 0x93, 0x24, 0x17, 0x52, 0x1c, 0xa2, 0x4f, 0x2b, 0x04, 0x59, 0xfe, 0x7e, 0x6e, 0x0b, + 0x09, 0x03, 0x39, 0xec, 0x0a, 0xa6, 0xfa, 0xef, 0xd5, 0xcc, 0xc2, 0xc6, 0xf4, 0xce, 0x8e, 0x94, + 0x1e, 0x36, 0xb2, 0x6b, 0xd1, 0xeb, 0xc6, 0x70, 0xd1, 0xbd, 0x1d, 0x66, 0x56, 0x20, 0xab, 0xf7, + 0x4f, 0x78, 0xa7, 0xf6, 0xd2, 0x98, 0x09, 0x58, 0x5a, 0x97, 0xda, 0xec, 0x58, 0xc6, 0xb0, 0x50 }; +#elif defined(AES128) + uint8_t key[16] = { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; + uint8_t in[64] = { 0x87, 0x4d, 0x61, 0x91, 0xb6, 0x20, 0xe3, 0x26, 0x1b, 0xef, 0x68, 0x64, 0x99, 0x0d, 0xb6, 0xce, + 0x98, 0x06, 0xf6, 0x6b, 0x79, 0x70, 0xfd, 0xff, 0x86, 0x17, 0x18, 0x7b, 0xb9, 0xff, 0xfd, 0xff, + 0x5a, 0xe4, 0xdf, 0x3e, 0xdb, 0xd5, 0xd3, 0x5e, 0x5b, 0x4f, 0x09, 0x02, 0x0d, 0xb0, 0x3e, 0xab, + 0x1e, 0x03, 0x1d, 0xda, 0x2f, 0xbe, 0x03, 0xd1, 0x79, 0x21, 0x70, 0xa0, 0xf3, 0x00, 0x9c, 0xee }; +#endif + uint8_t iv[16] = { 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff }; + uint8_t out[64] = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a, + 0xae, 0x2d, 0x8a, 0x57, 0x1e, 0x03, 0xac, 0x9c, 0x9e, 0xb7, 0x6f, 0xac, 0x45, 0xaf, 0x8e, 0x51, + 0x30, 0xc8, 0x1c, 0x46, 0xa3, 0x5c, 0xe4, 0x11, 0xe5, 0xfb, 0xc1, 0x19, 0x1a, 0x0a, 0x52, 0xef, + 0xf6, 0x9f, 0x24, 0x45, 0xdf, 0x4f, 0x9b, 0x17, 0xad, 0x2b, 0x41, 0x7b, 0xe6, 0x6c, 0x37, 0x10 }; + struct AES_ctx ctx; + + AES_init_ctx_iv(&ctx, key, iv); + AES_CTR_xcrypt_buffer(&ctx, in, 64); + + printf("CTR %s: ", xcrypt); + + if (0 == memcmp((char *) out, (char *) in, 64)) { + printf("SUCCESS!\n"); + return(0); + } else { + printf("FAILURE!\n"); + return(1); + } +} + + +static int test_decrypt_ecb(void) +{ +#if defined(AES256) + uint8_t key[] = { 0x60, 0x3d, 0xeb, 0x10, 0x15, 0xca, 0x71, 0xbe, 0x2b, 0x73, 0xae, 0xf0, 0x85, 0x7d, 0x77, 0x81, + 0x1f, 0x35, 0x2c, 0x07, 0x3b, 0x61, 0x08, 0xd7, 0x2d, 0x98, 0x10, 0xa3, 0x09, 0x14, 0xdf, 0xf4 }; + uint8_t in[] = { 0xf3, 0xee, 0xd1, 0xbd, 0xb5, 0xd2, 0xa0, 0x3c, 0x06, 0x4b, 0x5a, 0x7e, 0x3d, 0xb1, 0x81, 0xf8 }; +#elif defined(AES192) + uint8_t key[] = { 0x8e, 0x73, 0xb0, 0xf7, 0xda, 0x0e, 0x64, 0x52, 0xc8, 0x10, 0xf3, 0x2b, 0x80, 0x90, 0x79, 0xe5, + 0x62, 0xf8, 0xea, 0xd2, 0x52, 0x2c, 0x6b, 0x7b }; + uint8_t in[] = { 0xbd, 0x33, 0x4f, 0x1d, 0x6e, 0x45, 0xf2, 0x5f, 0xf7, 0x12, 0xa2, 0x14, 0x57, 0x1f, 0xa5, 0xcc }; +#elif defined(AES128) + uint8_t key[] = { 0x2b, 0x7e, 0x15, 0x16, 0x28, 0xae, 0xd2, 0xa6, 0xab, 0xf7, 0x15, 0x88, 0x09, 0xcf, 0x4f, 0x3c }; + uint8_t in[] = { 0x3a, 0xd7, 0x7b, 0xb4, 0x0d, 0x7a, 0x36, 0x60, 0xa8, 0x9e, 0xca, 0xf3, 0x24, 0x66, 0xef, 0x97 }; +#endif + + uint8_t out[] = { 0x6b, 0xc1, 0xbe, 0xe2, 0x2e, 0x40, 0x9f, 0x96, 0xe9, 0x3d, 0x7e, 0x11, 0x73, 0x93, 0x17, 0x2a }; + struct AES_ctx ctx; + + AES_init_ctx(&ctx, key); + AES_ECB_decrypt(&ctx, in); + + printf("ECB decrypt: "); + + if (0 == memcmp((char*) out, (char*) in, 16)) { + printf("SUCCESS!\n"); + return(0); + } else { + printf("FAILURE!\n"); + return(1); + } +} + + diff --git a/dependencies/transmit/src/tiny-AES-c/test.cpp b/dependencies/transmit/src/tiny-AES-c/test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b7c83048324f93cbbf32f090e4c19532a642b869 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/test.cpp @@ -0,0 +1,2 @@ +#include "aes.hpp" +#include "test.c" diff --git a/dependencies/transmit/src/tiny-AES-c/test_package/CMakeLists.txt b/dependencies/transmit/src/tiny-AES-c/test_package/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..2e4b2434c9d182e58c6f39dfc67d4a6b86957dce --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/test_package/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 2.8.12) +project(TinyAesPackageTest C CXX) + +include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) +conan_basic_setup() + +add_executable(example ../test.c) +add_executable(example_cpp ../test.cpp) + +target_link_libraries(example ${CONAN_LIBS}) +target_link_libraries(example_cpp ${CONAN_LIBS}) \ No newline at end of file diff --git a/dependencies/transmit/src/tiny-AES-c/test_package/conanfile.py b/dependencies/transmit/src/tiny-AES-c/test_package/conanfile.py new file mode 100644 index 0000000000000000000000000000000000000000..08f550fe13d2fdd4ede8277400c1586de273cff8 --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/test_package/conanfile.py @@ -0,0 +1,17 @@ +from conans import ConanFile, CMake, tools +import os + + +class TinyAesCTestConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + generators = "cmake" + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def test(self): + if not tools.cross_building(self.settings): + os.chdir("bin") + self.run(".%sexample" % os.sep) diff --git a/dependencies/transmit/src/tiny-AES-c/unlicense.txt b/dependencies/transmit/src/tiny-AES-c/unlicense.txt new file mode 100644 index 0000000000000000000000000000000000000000..68a49daad8ff7e35068f2b7a97d643aab440eaec --- /dev/null +++ b/dependencies/transmit/src/tiny-AES-c/unlicense.txt @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/dependencies/transmit/src/transmit_client.cpp b/dependencies/transmit/src/transmit_client.cpp new file mode 100644 index 0000000000000000000000000000000000000000..795f8b8c50c383d342f203ab0462ac97e3fee810 --- /dev/null +++ b/dependencies/transmit/src/transmit_client.cpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) Aisbench. 2020-2022. All rights reserved. + * description: + * + */ + +#include "transmit/transmit_client.h" +#include "aesutils/aesutils.h" +#include "log.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const int RET_OK = 0; +const int RET_RROR_1 = 1; +const int RET_RROR_2 = 2; +const int RET_RROR_3 = 3; +const int INT_NUMBER_16 = 16; +const int INT_NUMBER_10000 = 10000; +const int INT_NUMBER_20000 = 20000; + +Transmit_client::Transmit_client() +{ + aeskey = AESPUBLICKEY; + salt = SALT; +} + +Transmit_client::~Transmit_client() {} + +const uint64_t DEFAULT_TAG = 0x01020304; +const uint64_t MAX_LENGTH = 0x00100000; // 1M +#pragma pack(1) +using Tlv = struct { + uint32_t tag; + uint32_t len; + uint8_t* buf; +}; +#pragma pack() + +int SendAll(int fd, uint8_t* buf, int len) +{ + uint8_t* ptr = buf; + int unSend = len; + while (unSend > 0) { + int ret = send(fd, ptr, unSend, 0); + if (ret < 0) { + return -1; + } else { + ptr += ret; + unSend = unSend - ret; + } + } + return 0; +} + +int GetDynimicPort(int oriPort) +{ + char* dystr = getenv("AIS_BENCH_CON_PORT"); + if (dystr == nullptr) { + return oriPort; + } + int dyport = atoi(dystr); + if (dyport >= INT_NUMBER_10000 && dyport <= INT_NUMBER_20000) { + return dyport; + } else { + return oriPort; + } +} + +std::string GetDynamicIp() +{ + char* dystr = getenv("AIS_BENCH_HOST_IP"); + if (dystr == nullptr) { + return SERVER_ADDRESS; + } + return std::string(dystr); +} + +int Transmit_client::SentMsgToServer(const std::string& msg) +{ + if (msg.empty()) { + ERROR("invalid msg! null"); + return 1; + } + Tlv frame; + frame.tag = htonl(DEFAULT_TAG); + frame.len = msg.length(); + uint32_t trim = INT_NUMBER_16 - (frame.len % INT_NUMBER_16); + if (trim != 0) { + frame.len += trim; + } + frame.buf = new uint8_t[frame.len]; + memset(frame.buf, 0, frame.len); + memcpy(frame.buf, msg.c_str(), msg.length()); + + + // preprocess before sending + AesUtils::encrypt_cbc_256(frame.buf, frame.len); + + int ret; + // 创建客户端套节字 + int sClient = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + struct sockaddr_in server; + memset(&server, 0, sizeof(sockaddr_in)); + server.sin_family = PF_INET; + int port = GetDynimicPort(SERVERPORT); + std::string ip = GetDynamicIp(); + server.sin_port = htons(port); + server.sin_addr.s_addr = inet_addr(ip.c_str()); + + int keepalive = true; + if (setsockopt(sClient, SOL_SOCKET, SO_KEEPALIVE, (void*)&keepalive, sizeof(keepalive))) { + WARN("set socketopt keepalive failed ret"); + return RET_RROR_2; + } + + ret = connect(sClient, (struct sockaddr*)&server, sizeof(sockaddr_in)); + if (0 != ret) { + ERROR("connect to server failure！"); + close(sClient); + delete[] frame.buf; + frame.buf = nullptr; + return RET_RROR_3; + } + + ret = SendAll(sClient, (uint8_t*)&frame.tag, sizeof(uint32_t)); + if (ret == 0) { + int length = frame.len; + frame.len = htonl(frame.len); + ret = SendAll(sClient, (uint8_t*)&frame.len, sizeof(uint32_t)); + if (ret == 0) { + ret = SendAll(sClient, frame.buf, length); + } + } + + // 释放连接和进行结束工作 + close(sClient); + delete[] frame.buf; + frame.buf = nullptr; + return ret; +} diff --git a/dependencies/transmit/src/transmit_server.cpp b/dependencies/transmit/src/transmit_server.cpp new file mode 100644 index 0000000000000000000000000000000000000000..835bf78d7d8fb8b053c2605dceea7e0dd00d81e1 --- /dev/null +++ b/dependencies/transmit/src/transmit_server.cpp @@ -0,0 +1,270 @@ +/* + * Copyright (c) Aisbench. 2020-2022. All rights reserved. + * Description: + * Author: + * Create: + */ + +#include "transmit/transmit_server.h" +#include "aesutils/aesutils.h" +#include "log.h" +#include +#include "string.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + + +const uint8_t RECV_SUCCESS = 0; +const uint8_t RECV_FAILED = -1; +const uint8_t RET_OK = 0; +const uint8_t RET_ERROR_1 = 1; +const uint8_t RET_ERROR_2 = 2; +const uint8_t RET_ERROR_3 = 3; +const uint8_t INT_CONST_NUMBER_5 = 5; + +void Transmit(Transmit_server* server) +{ + server->Worker(); +} + +Transmit_server::Transmit_server() +{ + aeskey = AESPUBLICKEY; + salt = SALT; + frame.buf = nullptr; +} + +Transmit_server::~Transmit_server() +{ + Stop(); +} + +int GetDynimicPort(int oriPort) +{ + char* dystr = getenv("AIS_BENCH_CON_PORT"); + if (dystr == nullptr) { + return oriPort; + } + int dyport = atoi(dystr); + int startPort = 10000; + int endPort = 20000; + if (dyport >= startPort && dyport <= endPort) { + return dyport; + } else { + return oriPort; + } +} + +int Transmit_server::ConnectInit() +{ + // 创建套接字 + ser_sock = socket(AF_INET, SOCK_STREAM, 0); + if (ser_sock == -1) { + ERROR("Transmit_server create socket failed！"); + return RET_ERROR_1; + } + + // 防止旧socket处于time_wait状态导致监听失败 + int opt = 1; + setsockopt(ser_sock, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + + int keepalive = true; + if (setsockopt(ser_sock, SOL_SOCKET, SO_KEEPALIVE, reinterpret_cast(&keepalive), sizeof(keepalive))) { + WARN("set socketopt keepalive failed ret"); + return RET_ERROR_1; + } + + // 套接字绑定ip及端口号 + char ip[] = "0.0.0.0"; + sockaddr_in ser_addr; + ser_addr.sin_family = AF_INET; + ser_addr.sin_addr.s_addr = inet_addr(ip); + int port = GetDynimicPort(SERVERPORT); + ser_addr.sin_port = htons(port); + if (bind(ser_sock, reinterpret_cast(&ser_addr), sizeof(ser_addr)) == -1) { + close(ser_sock); + ser_sock = -1; + ERROR("Transmit_server socket bind failed port:%d", port); + return RET_ERROR_2; + } + + // 开始监听 + INFO("Transmit_server start listen %s : %d", ip, port); + if (listen(ser_sock, INT_CONST_NUMBER_5) == -1) { + close(ser_sock); + ser_sock = -1; + ERROR("Transmit_server listen() failed！"); + return RET_ERROR_3; + } + + return RET_OK; +} + +int Transmit_server::Worker() +{ + + while (true) { + if (ser_sock <= 0) { + continue; + } + // 监听客户端数据传输 + sockaddr_in clnt_addr; + socklen_t addr_length = sizeof(clnt_addr); + clnt_sock = accept(ser_sock, reinterpret_cast(&clnt_addr), &addr_length); + if (clnt_sock == -1) { + continue; + } + + // 接收数据 + if (ReceiveMsg() != RECV_SUCCESS) { + ERROR("Transmit_server post-process client msg failed: %d", clnt_sock); + } + // 必须要有返回数据， 这样才算一个完整的请求 + close(clnt_sock); + clnt_sock = -1; + } + return 0; +} + +int Transmit_server::PostprocessData(uint8_t* input, uint32_t length) +{ + int ret = AesUtils::decrypt_cbc_256(input, length); + if (ret != 0) { + ERROR("Transmit_server decrypt client msg failure！"); + return RECV_FAILED; + } + // save to given path + callBack(reinterpret_cast(input)); + + return RECV_SUCCESS; +} + + +/* Read "n" bytes from a descriptor. */ +ssize_t Readn(int fd, void* vptr, size_t n) +{ + size_t nleft; + ssize_t nread; + char* ptr; + + ptr = reinterpret_cast(vptr); + nleft = n; + + while (nleft > 0) { + if ((nread = read(fd, ptr, nleft)) < 0) { + if (errno == EINTR) { + nread = 0; /* and call read() again */ + } else { + return(-1); + } + } else if (nread == 0) { + break; /* EOF */ + } + + nleft -= nread; + ptr += nread; + } + + return(n - nleft); /* return >= 0 */ +} + +int Transmit_server::ReceiveMsg() +{ + int nRead = Readn(clnt_sock, reinterpret_cast(&frame.tag), sizeof(uint32_t)); + if (nRead < 0) { + return RECV_FAILED; + } else { + frame.tag = ntohl(frame.tag); + } + if (frame.tag != DEFAULT_TAG) { + return RECV_FAILED; + } + + nRead = Readn(clnt_sock, reinterpret_cast(&frame.len), sizeof(uint32_t)); + if (nRead < 0) { + return RECV_FAILED; + } else { + frame.len = ntohl(frame.len); + } + + if (frame.len > MAX_LENGTH) { + return RECV_FAILED; + } + + frame.buf = new uint8_t[frame.len]; + nRead = Readn(clnt_sock, frame.buf, frame.len); + if (nRead < 0) { + return RECV_FAILED; + } + + // 解密 + int ret = PostprocessData(frame.buf, frame.len); + delete[] frame.buf; + frame.buf = nullptr; + return ret; +} + +int Transmit_server::Start() +{ + int ret = ConnectInit(); + if (ret != 0) { + Stop(); + return -1; + } + thread t(Transmit, this); + transmitThreadID = t.native_handle(); + + t.detach(); + + DEBUG("Transmit_server thread started！"); + return 0; +} + +bool Transmit_server::StopTransmitThread() +{ + int ret = pthread_cancel(transmitThreadID); + if (0 == ret) { + CleanTCPResource(); + } + return (ret == 0) ? true : false; +} + +void Transmit_server::CleanTCPResource() +{ + if (ser_sock <= 0 && clnt_sock <= 0) { + return; + } + + if (ser_sock > 0) { + close(ser_sock); + ser_sock = 0; + } + if (clnt_sock > 0) { + close(clnt_sock); + clnt_sock = 0; + } + INFO("Transmit_server resource is released!"); +} + +int Transmit_server::Stop() +{ + StopTransmitThread(); + CleanTCPResource(); + if (frame.buf != nullptr) { + delete[] frame.buf; + } + sleep(1); + + return 0; +} diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000000000000000000000000000000000..e6ca1760ccd663106d8c7e66e6a6290a618c773a --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from setuptools import setup, find_packages +import sys +import platform + +setup( + name="ais_bench_logging", + version="2.1", + description="ais_bench logging", + packages=find_packages(), + include_package_data=True, + install_requires=[ + "wheel", + ], + package_data={ "ais_bench.logging":["libais_bench_logging.so"], }, + data_files=[ ("ais_bench/logging", ["ais_bench/logging/libais_bench_logging.so"]) ], + options={ + "bdist_wheel": { + "plat_name": f"{sys.platform}-{platform.machine()}" + } + } +) \ No newline at end of file diff --git a/test.sh b/test.sh new file mode 100644 index 0000000000000000000000000000000000000000..8854b43c71d9c161109bc58f315a51329efff29d --- /dev/null +++ b/test.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +CUR_PATH=$(cd "$(dirname "$0")";pwd) +TEST_PATH=${CUR_PATH}/test + + +[ -f ${TEST_PATH}/build/logging_test ] && ${TEST_PATH}/build/logging_test \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..d8cd5e5c517c28df3ccdaff0e61ebc8485bed9b9 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,49 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 设置最小CMake版本 +cmake_minimum_required(VERSION 3.10) + +# 设置项目名和版本 +project(logging_test) + +# 设置C++标准 +set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +set(CMAKE_CXX_FLAGS "-fPIC -std=c++14 -O0 -Wall -g2 ${CMAKE_CXX_FLAGS}") +add_compile_options(-Wno-format-extra-args) +add_compile_options(-fstack-protector-strong -Wl,-z,now -D_FORTIFY_SOURCE=2 -O2) +set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-z,relro,-z,now") +SET(CMAKE_SKIP_RPATH TRUE) + +set(LOGGING_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../ais_bench/logging) + +# 添加源文件到项目中 +aux_source_directory(${LOGGING_SRC_DIR}/src DIR_SRCS) +aux_source_directory(${CMAKE_CURRENT_SOURCE_DIR}/. DIR_SRCS) + +# 指定头文件路径 +include_directories(${LOGGING_SRC_DIR}/include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) +link_directories(${LOGGING_SRC_DIR}/lib) + + +# 添加源代码文件 +add_executable(logging_test ${DIR_SRCS}) + + +# 链接 Google Test 库 +target_link_libraries(logging_test gtest pthread dl util) +target_link_libraries(logging_test gtest stdc++fs libtransmit.a) diff --git a/test/common.cpp b/test/common.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d3d7152b439ccf602ffa36978a3d3c5000d811d1 --- /dev/null +++ b/test/common.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include "common.h" +#include "time.h" + +std::chrono::system_clock::time_point ConstructTimePoint(int year, int month, int day, int hour, int minute, int sec) { + std::tm tm = {0}; + tm.tm_year = year - 1900; // 年份 + tm.tm_mon = month - 1; // 月份，注意月份是从0开始的 + tm.tm_mday = day; // 日期 + tm.tm_hour = hour; // 小时 + tm.tm_min = minute; // 分钟 + tm.tm_sec = sec; // 秒 + + std::time_t tt = std::mktime(&tm); + auto tp = std::chrono::system_clock::from_time_t(tt); + return tp; +} \ No newline at end of file diff --git a/test/include/common.h b/test/include/common.h new file mode 100644 index 0000000000000000000000000000000000000000..5f05987ecce4cb2fe44586af86c715380a458d38 --- /dev/null +++ b/test/include/common.h @@ -0,0 +1,24 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#ifndef COMMON_H +#define COMMON_H + + +std::chrono::system_clock::time_point ConstructTimePoint(int year, int month, int day, int hour, int minute, int sec); + + +#endif // COMMON_H \ No newline at end of file diff --git a/test/main.cpp b/test/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..028897c201fe4e7104077012a2fc5819e8da4cbc --- /dev/null +++ b/test/main.cpp @@ -0,0 +1,20 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} \ No newline at end of file diff --git a/test/test_collector.cpp b/test/test_collector.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cb959a8c8317a32ee3998984142cd863f816a1f7 --- /dev/null +++ b/test/test_collector.cpp @@ -0,0 +1,108 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include "nlohmann/json.hpp" +#define private public +#define protected public + +#include "Collector.h" + +#undef private +#undef protected + +void RemoveExisting(const std::string& fileName) { + std::experimental::filesystem::path filePath = std::experimental::filesystem::current_path() / std::experimental::filesystem::path(fileName); + if (std::experimental::filesystem::exists(filePath)) { + std::experimental::filesystem::remove(filePath); + } + return; +} + +json NonFormatedJson = json({"key", "value"}); + +json FormatedEarlyJson = json({ + {"key", {{"value", "value_early"}, {"time", 20}}}, + {"key_new", {{"value", "value_early"}, {"time", 30}}}, + {"key_start", {{"value", "value_early"}, {"time", 40}}}, +}); + +json FormatedLaterJson = json({ + {"key", {{"value", "value_later"}, {"time", 120}}}, + {"key_start", {{"value", "value_later"}, {"time", 140}}}, +}); + +json MergedSuccessJson = json({ + {"key", {{"value", "value_later"}, {"time", 120}}}, + {"key_new", {{"value", "value_early"}, {"time", 30}}}, + {"key_start", {{"value", "value_early"}, {"time", 40}}}, +}); + +void CreateJson(const std::string& fileName, json jsonData) { + RemoveExisting(fileName); + std::ofstream outFile(fileName); + outFile << jsonData; +} + + + +TEST(Collector, merge_file_success) +{ + std::string jsonFile1 = "data1.json"; + std::string jsonFile2 = "data2.json"; + + CreateJson(jsonFile1, FormatedEarlyJson); + CreateJson(jsonFile2, FormatedLaterJson); + + Collector collector = Collector(); + + json mergedJson; + collector.Collect({jsonFile1, jsonFile2}, mergedJson); + EXPECT_TRUE(mergedJson==MergedSuccessJson); +} + +TEST(Collector, merge_file_partially_failed) +{ + std::string jsonFile1 = "data1.json"; + std::string jsonFile2 = "data2.json"; + + CreateJson(jsonFile1, FormatedEarlyJson); + CreateJson(jsonFile2, NonFormatedJson); + + Collector collector = Collector(); + + json mergedJson; + collector.Collect({jsonFile1, jsonFile2}, mergedJson); + EXPECT_TRUE(mergedJson==FormatedEarlyJson); +} + +TEST(Collector, merge_file_all_failed) +{ + std::string jsonFile1 = "data1.json"; + std::string jsonFile2 = "data2.json"; + + CreateJson(jsonFile1, NonFormatedJson); + CreateJson(jsonFile2, NonFormatedJson); + + Collector collector = Collector(); + + json mergedJson; + json emptyJson; + collector.Collect({jsonFile1, jsonFile2}, mergedJson); + EXPECT_TRUE(mergedJson==emptyJson); +} + + diff --git a/test/test_recorder.cpp b/test/test_recorder.cpp new file mode 100644 index 0000000000000000000000000000000000000000..a6deb9daabf8e58d9fa9da5076f668ef9425f708 --- /dev/null +++ b/test/test_recorder.cpp @@ -0,0 +1,244 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#define private public +#define protected public + +#include "Recorder.h" + +#undef private +#undef protected +#include "time.h" +#include "common.h" + +#define ABS_DIFF(a, b) ((a) > (b) ? (a) - (b) : (b) - (a)) + +TEST(Recorder, constructor) +{ + Recorder recorderUndefined{TaskType::UNDEFINED}; + Recorder recorderTraining{TaskType::TRAINING}; + Recorder recorderInference{TaskType::INFERENCE}; + + EXPECT_TRUE(recorderUndefined.m_durationEvent.empty()); + EXPECT_TRUE(recorderTraining.m_durationEvent==m_trainingDurationEventList); + EXPECT_TRUE(recorderInference.m_durationEvent==m_inferenceDurationEventList); +} + + +TEST(Recorder, set_task_type) +{ + Recorder recorder{TaskType::UNDEFINED}; + EXPECT_TRUE(recorder.m_durationEvent.empty()); + recorder.SetTaskType(TaskType::INFERENCE); + EXPECT_TRUE(recorder.m_durationEvent==m_inferenceDurationEventList); + recorder.SetTaskType(TaskType::TRAINING); + EXPECT_TRUE(recorder.m_durationEvent==m_trainingDurationEventList); +} + +TEST(Recorder, record_event_success) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacket{ + "average_power", + 0, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::POINT, dataPacket); + EXPECT_TRUE(!recorder.m_pointRecording.empty()); +} + +TEST(Recorder, record_start_success) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacket{ + "train", + 0, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::START, dataPacket); + EXPECT_TRUE(!recorder.m_startRecording.empty()); +} + +TEST(Recorder, record_end_success) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacketStart{ + "train", + 1, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::START, dataPacketStart); + DataPacket dataPacketEnd{ + "train", + 1, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 50) + }; + recorder.Record(EventType::END, dataPacketEnd); + EXPECT_TRUE(!recorder.m_endRecording.empty()); +} + + +TEST(Recorder, record_event_failed_unknown_name) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacket{ + "average_po", + 0, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::POINT, dataPacket); + EXPECT_FALSE(!recorder.m_pointRecording.empty()); +} + +TEST(Recorder, record_start_failed_unknown_name) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacket{ + "trainxx", + 0, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::START, dataPacket); + EXPECT_FALSE(!recorder.m_startRecording.empty()); +} + +TEST(Recorder, record_start_failed_repeat) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacket{ + "train", + 0, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::START, dataPacket); + recorder.Record(EventType::START, dataPacket); + EXPECT_EQ(recorder.m_startRecording.size(), 1); +} + +TEST(Recorder, record_end_failed_not_in_start) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacket{ + "trainxx", + 0, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::END, dataPacket); + EXPECT_FALSE(!recorder.m_endRecording.empty()); +} + +TEST(Recorder, record_end_failed_sample_num_not_match) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacketStart{ + "train", + 12, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::START, dataPacketStart); + DataPacket dataPacketEnd{ + "train", + 13, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 30) + }; + recorder.Record(EventType::END, dataPacketEnd); + EXPECT_FALSE(!recorder.m_endRecording.empty()); +} + +TEST(Recorder, record_end_failed_time_before_start) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacketStart{ + "train", + 12, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 30) + }; + recorder.Record(EventType::START, dataPacketStart); + DataPacket dataPacketEnd{ + "train", + 12, + "1000", + ConstructTimePoint(2020, 10, 1, 12, 0, 0) + }; + recorder.Record(EventType::END, dataPacketEnd); + EXPECT_FALSE(!recorder.m_endRecording.empty()); +} + +TEST(Recorder, training_whole_process) +{ + Recorder recorder{TaskType::TRAINING}; + DataPacket dataPacket; + dataPacket = {"train", 1000, "", ConstructTimePoint(2020, 10, 1, 12, 0, 10)}; + recorder.Record(EventType::START, dataPacket); + dataPacket = {"train", 1000, "", ConstructTimePoint(2020, 10, 1, 12, 0, 20)}; + recorder.Record(EventType::END, dataPacket); + dataPacket = {"average_power", 1, "100w", ConstructTimePoint(2020, 10, 1, 12, 0, 35)}; + recorder.Record(EventType::POINT, dataPacket); + + json outJson = recorder.GetOutJson(); + ASSERT_TRUE(outJson.is_object()); + + ASSERT_TRUE(outJson.contains("train_start_time")); + ASSERT_TRUE(outJson.contains("train_end_time")); + ASSERT_TRUE(outJson.contains("average_power")); + ASSERT_TRUE(outJson.contains("throughput_ratio")); + + EXPECT_EQ(outJson["train_start_time"]["value"], "2020-10-01 12:00:10.000"); + EXPECT_EQ(outJson["train_end_time"]["value"], "2020-10-01 12:00:20.000"); + + EXPECT_EQ(outJson["average_power"]["value"], "100w"); + EXPECT_LT(ABS_DIFF(std::stod(std::string(outJson["throughput_ratio"]["value"])), 100.0), 0.1); + +} + +TEST(Recorder, inference_whole_process) +{ + Recorder recorder{TaskType::INFERENCE}; + DataPacket dataPacket; + dataPacket = {"infer", 1000, "", ConstructTimePoint(2020, 10, 1, 12, 0, 10)}; + recorder.Record(EventType::START, dataPacket); + dataPacket = {"infer", 1000, "", ConstructTimePoint(2020, 10, 1, 12, 0, 20)}; + recorder.Record(EventType::END, dataPacket); + dataPacket = {"accuracy", 1, "80%", ConstructTimePoint(2020, 10, 1, 12, 0, 35)}; + recorder.Record(EventType::POINT, dataPacket); + + json outJson = recorder.GetOutJson(); + ASSERT_TRUE(outJson.is_object()); + + ASSERT_TRUE(outJson.contains("infer_latency")); + ASSERT_TRUE(outJson.contains("accuracy")); + ASSERT_TRUE(outJson.contains("throughput_ratio")); + + EXPECT_LT(ABS_DIFF(std::stod(std::string(outJson["infer_latency"]["value"])), 10.0), 0.1); + + EXPECT_EQ(outJson["accuracy"]["value"], "80%"); + EXPECT_LT(ABS_DIFF(std::stod(std::string(outJson["throughput_ratio"]["value"])), 100.0), 0.1); + +} + + + + diff --git a/test/test_utils.cpp b/test/test_utils.cpp new file mode 100644 index 0000000000000000000000000000000000000000..7fe879e59c719249ca6d4fb5618bfa517a2120b7 --- /dev/null +++ b/test/test_utils.cpp @@ -0,0 +1,121 @@ +// Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#define private public +#define protected public + +#include "Utils.h" + +#undef private +#undef protected +#include "common.h" + + +TEST(check_format, success) +{ + json jsonData = json({ + {"key1", {{"value", "value_string"}, {"time", 42}}}, + {"key2", {{"value", "value_string"}, {"time", 42}}}, + {"key3", {{"value", "value_string"}, {"time", 42}}}, + }); + + EXPECT_TRUE(Utils::CheckFormat(jsonData)); +} + +TEST(check_format, failed_not_object) +{ + json jsonData = json(1); + + EXPECT_TRUE(!Utils::CheckFormat(jsonData)); +} + +TEST(check_format, failed_value_not_object) +{ + json jsonData = json({ + {"key1", 12}, + {"key2", 12}, + {"key3", 12}, + }); + + EXPECT_TRUE(!Utils::CheckFormat(jsonData)); +} + + +TEST(check_format, failed_not_contain_value_attr) +{ + json jsonData = json({ + {"key1", {"time", 42}}, + {"key2", {"time", 42}}, + {"key3", {"time", 42}}, + }); + + EXPECT_TRUE(!Utils::CheckFormat(jsonData)); +} + +TEST(check_format, failed_value_attr_not_string) +{ + json jsonData = json({ + {"key1", {{"value", 12}, {"time", 42}}}, + {"key2", {{"value", 12}, {"time", 42}}}, + {"key3", {{"value", 12}, {"time", 42}}}, + }); + + EXPECT_TRUE(!Utils::CheckFormat(jsonData)); +} + +TEST(check_format, failed_not_contain_time_attr) +{ + json jsonData = json({ + {"key1", {"value", "value_string"}}, + {"key2", {"value", "value_string"}}, + {"key3", {"value", "value_string"}}, + }); + + EXPECT_TRUE(!Utils::CheckFormat(jsonData)); +} + +TEST(check_format, failed_time_attr_not_int) +{ + json jsonData = json({ + {"key1", {{"value", "value_string"}, {"time", "42"}}}, + {"key2", {{"value", "value_string"}, {"time", "42"}}}, + {"key3", {{"value", "value_string"}, {"time", "42"}}}, + }); + + EXPECT_TRUE(!Utils::CheckFormat(jsonData)); +} + + +TEST(time_to_string, success) +{ + auto tp = ConstructTimePoint(2020, 10, 1, 12, 0, 0); + auto timeStr = Utils::TimeToString(tp); + EXPECT_EQ(timeStr, "2020-10-01 12:00:00.000"); +} + +TEST(time_to_int64_milliseconds, success) +{ + auto tp = ConstructTimePoint(1970, 1, 1, 8, 0, 0); + auto milliseconds = Utils::TimeToInt64Millisec(tp); + EXPECT_EQ(milliseconds, 0); +} + +TEST(calculate_time_diff, success) +{ + auto tp1 = ConstructTimePoint(2020, 10, 1, 12, 0, 0); + auto tp2 = ConstructTimePoint(2020, 10, 1, 12, 0, 10); + auto milliseconds = Utils::CalculateMillisecondsDifference(tp1, tp2); + EXPECT_EQ(milliseconds, 10000); +}