diff --git a/aiframe/CMakeLists.txt b/aiframe/CMakeLists.txt index 8ece1ce61e864666bd80288e17398c16a4723fed..9f8022f59bd8d2c66926a107b42571caeb8bd2d0 100644 --- a/aiframe/CMakeLists.txt +++ b/aiframe/CMakeLists.txt @@ -4,8 +4,6 @@ project(ONNXRunner) set(CMAKE_CXX_STANDARD 17) -#set(CMAKE_CXX_FLAGS_DEBUG "-g -Wall") - set(INC_DIR /usr/include) set(INC_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/include) set(LIB_DIR /usr/lib64) @@ -18,6 +16,6 @@ link_directories(${LIB_DIR}) add_library(ONNXRunner SHARED ONNXRunner.cpp) -#target_link_libraries(ONNXRunner -#PRIVATE -#libonnxruntime.so) +target_link_libraries(ONNXRunner +PRIVATE +libcrypto.so) # libonnxruntime.so diff --git a/aiframe/ONNXRunner.cpp b/aiframe/ONNXRunner.cpp index 3ec159c728d184f6a2ee7be565998d30d7a7cabe..3aefae330ad8634786b69ecaf0e767cb0da55b52 100644 --- a/aiframe/ONNXRunner.cpp +++ b/aiframe/ONNXRunner.cpp @@ -1,151 +1,244 @@ -#include -#include +#include "include/ONNXRunner.h" #include +#include #include -#include "include/ONNXRunner.h" +#include -namespace boltONNXRunner{ +namespace compilerONNXRunner { -Ort::Value ONNXRunner::getInputValueFloat(Ort::Session *session, +Ort::Value ONNXRunner::getInputValueFloat(Ort::Session *session, std::vector &input, - int inputIdx) { - auto typeInfo = session->GetInputTypeInfo(inputIdx); - auto tensorInfo = typeInfo.GetTensorTypeAndShapeInfo(); - auto inputDims = tensorInfo.GetShape(); - std::replace_if( - inputDims.begin(), inputDims.end(), [](int64_t &i) { return i < 0; }, 1); - - size_t inputTensorSize = std::accumulate(inputDims.begin(), inputDims.end(), - 1, std::multiplies()); - auto memory_info = - Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); - auto inputTmp = Ort::Value::CreateTensor( - memory_info, input.data(), inputTensorSize, inputDims.data(), - inputDims.size()); - auto inputTensor = &inputTmp; - return inputTmp; + int inputIdx, int batchSize) { + auto typeInfo = session->GetInputTypeInfo(inputIdx); + auto tensorInfo = typeInfo.GetTensorTypeAndShapeInfo(); + auto inputDims = tensorInfo.GetShape(); + std::replace_if( + inputDims.begin(), inputDims.end(), [](int64_t &i) { return i < 0; }, 1); + + size_t inputTensorSize = std::accumulate(inputDims.begin(), inputDims.end(), + 1, std::multiplies()); + // try to add batch size + inputDims[0] = batchSize; + inputTensorSize = inputTensorSize * batchSize; + auto memoryInfo = + Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + auto inputTmp = + Ort::Value::CreateTensor(memoryInfo, input.data(), inputTensorSize, + inputDims.data(), inputDims.size()); + auto inputTensor = &inputTmp; + return inputTmp; } -Ort::Value ONNXRunner::getInputValueString(Ort::AllocatorWithDefaultOptions allocator, - Ort::Session *session, - std::vector &input, - int inputIdx) { - auto typeInfo = session->GetInputTypeInfo(inputIdx); - auto tensorInfo = typeInfo.GetTensorTypeAndShapeInfo(); - auto inputDims = tensorInfo.GetShape(); - - std::replace_if( - inputDims.begin(), inputDims.end(), [](int64_t &i) { return i < 0; }, 1); - - size_t inputTensorSize = std::accumulate(inputDims.begin(), inputDims.end(), - 1, std::multiplies()); - const char* input_strings[inputTensorSize]; - for(int i = 0; i < inputTensorSize; i++) { - input_strings[i] = input[i].c_str(); - } - - auto memory_info = - Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); - auto inputTmp = Ort::Value::CreateTensor(allocator, inputDims.data(), - inputDims.size(), - ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING); - inputTmp.FillStringTensor(input_strings, inputTensorSize); - auto inputTensor = &inputTmp; - return inputTmp; +Ort::Value ONNXRunner::getInputValueString( + Ort::AllocatorWithDefaultOptions allocator, Ort::Session *session, + std::vector &input, int inputIdx, int batchSize) { + auto typeInfo = session->GetInputTypeInfo(inputIdx); + auto tensorInfo = typeInfo.GetTensorTypeAndShapeInfo(); + auto inputDims = tensorInfo.GetShape(); + + std::replace_if( + inputDims.begin(), inputDims.end(), [](int64_t &i) { return i < 0; }, 1); + + size_t inputTensorSize = std::accumulate(inputDims.begin(), inputDims.end(), + 1, std::multiplies()); + inputDims[0] = batchSize; + inputTensorSize = inputTensorSize * batchSize; + const char *inputStrings[inputTensorSize]; + for (int i = 0; i < inputTensorSize; i++) { + inputStrings[i] = input[i].c_str(); + } + + auto memoryInfo = + Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + auto inputTmp = + Ort::Value::CreateTensor(allocator, inputDims.data(), inputDims.size(), + ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING); + inputTmp.FillStringTensor(inputStrings, inputTensorSize); + auto inputTensor = &inputTmp; + return inputTmp; } -Ort::Value ONNXRunner::getInputValueInt64(Ort::Session *session, +Ort::Value ONNXRunner::getInputValueInt64(Ort::Session *session, std::vector &input, - int inputIdx) { - auto typeInfo = session->GetInputTypeInfo(inputIdx); - auto tensorInfo = typeInfo.GetTensorTypeAndShapeInfo(); - auto inputDims = tensorInfo.GetShape(); - std::replace_if( - inputDims.begin(), inputDims.end(), [](int64_t &i) { return i < 0; }, 1); - - size_t inputTensorSize = std::accumulate(inputDims.begin(), inputDims.end(), - 1, std::multiplies()); - auto memory_info = - Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); - auto inputTmp = Ort::Value::CreateTensor( - memory_info, input.data(), inputTensorSize, inputDims.data(), - inputDims.size()); - auto inputTensor = &inputTmp; - return inputTmp; + int inputIdx, int batchSize) { + auto typeInfo = session->GetInputTypeInfo(inputIdx); + auto tensorInfo = typeInfo.GetTensorTypeAndShapeInfo(); + auto inputDims = tensorInfo.GetShape(); + std::replace_if( + inputDims.begin(), inputDims.end(), [](int64_t &i) { return i < 0; }, 1); + + size_t inputTensorSize = std::accumulate(inputDims.begin(), inputDims.end(), + 1, std::multiplies()); + inputDims[0] = batchSize; + inputTensorSize = inputTensorSize * batchSize; + auto memoryInfo = + Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + auto inputTmp = Ort::Value::CreateTensor( + memoryInfo, input.data(), inputTensorSize, inputDims.data(), + inputDims.size()); + auto inputTensor = &inputTmp; + return inputTmp; } -float ONNXRunner::runONNXModel(std::vector input_string, std::vector input_int64, std::vector input_float){ - Ort::AllocatorWithDefaultOptions allocator; - - //Try to get input; - int input_count = session->GetInputCount(); - - //Get input name - std::vector inputNameList; - for (int i = 0; i < input_count; i++) { - auto inputName = session->GetInputNameAllocated(i, allocator); - auto inputNameStr = inputName.get(); - inputNameList.push_back(inputNameStr); - } - - //Form input tensor(s) - std::vector input_final; - std::vector inputNameStr_final; - - int currentIdx = 0; - if(!input_string.empty()) { - input_final.push_back(getInputValueString(allocator, session, input_string, currentIdx)); - currentIdx ++; - } - - if(!input_int64.empty()) { - input_final.push_back(getInputValueInt64(session, input_int64, currentIdx)); - currentIdx ++; - } - - if(!input_float.empty()) { - input_final.push_back(getInputValueFloat(session, input_float, currentIdx)); - currentIdx ++; +std::vector +ONNXRunner::runONNXModel(std::vector inputString, + std::vector inputInt64, + std::vector inputFloat, int batchSize) { + Ort::AllocatorWithDefaultOptions allocator; + + // Get input count + int inputCount = session->GetInputCount(); + + // Get input name + std::vector inputNameList; + for (int i = 0; i < inputCount; i++) { + auto inputName = session->GetInputNameAllocated(i, allocator); + auto inputNameStr = inputName.get(); + inputNameList.push_back(inputNameStr); + } + + // Form input tensor(s) + std::vector inputFinal; + std::vector inputNameStrFinal; + int currentIdx = 0; + if (!inputString.empty()) { + inputFinal.push_back(getInputValueString(allocator, session, inputString, + currentIdx, batchSize)); + currentIdx++; + } + + if (!inputInt64.empty()) { + inputFinal.push_back( + getInputValueInt64(session, inputInt64, currentIdx, batchSize)); + currentIdx++; + } + + if (!inputFloat.empty()) { + inputFinal.push_back( + getInputValueFloat(session, inputFloat, currentIdx, batchSize)); + currentIdx++; + } + + for (int i = 0; i < inputCount; i++) { + inputNameStrFinal.push_back(inputNameList[i].c_str()); + } + + // Run model + int outputCount = session->GetOutputCount(); + std::vector outputNameList; + for (int i = 0; i < outputCount; i++) { + auto outputName = session->GetOutputNameAllocated(i, allocator); + std::string outputNameStr = outputName.get(); + if (!outputNameStr.empty()) { + outputNameList.push_back(outputNameStr); + } else { + std::string outputNameDefault = "Output_" + std::to_string(i); + outputNameList.push_back(outputNameDefault); } + } + + std::vector outputNameStrFinal; + for (int i = 0; i < outputCount; i++) { + outputNameStrFinal.push_back(outputNameList[i].c_str()); + } + + auto outputTensors = session->Run( + Ort::RunOptions{nullptr}, inputNameStrFinal.data(), inputFinal.data(), + inputCount, outputNameStrFinal.data(), outputCount); + + // Get result and return + std::vector probs; + float *outputProbability = outputTensors[0].GetTensorMutableData(); + for (int i = 0; i < batchSize; i++) { + Ort::Value mapOut = + outputTensors[1].GetValue(static_cast(i), allocator); + Ort::Value keysOrt = mapOut.GetValue(0, allocator); + int64_t *keysRet = keysOrt.GetTensorMutableData(); + Ort::Value valuesOrt = mapOut.GetValue(1, allocator); + float *valuesRet = valuesOrt.GetTensorMutableData(); + probs.push_back((*(valuesRet + 1))); + } + + return probs; +} - for (int i = 0; i < input_count; i++) { - inputNameStr_final.push_back(inputNameList[i].c_str()); +int64_t ONNXRunner::runONNXModelOptimizer(std::vector inputString, + std::vector inputInt64, + std::vector inputFloat, + int batchSize) { + Ort::AllocatorWithDefaultOptions allocator; + + // Get input count + int inputCount = session->GetInputCount(); + std::vector inputInt64Tensor(FEATURE_SIZE_INT64_OPT); + std::vector inputStringTensor; + std::vector inputFinal; + + inputInt64.clear(); + inputInt64.resize(FEATURE_SIZE_INT64_OPT); + for (int i = 0; i < FEATURE_SIZE_INT64_OPT; i++) { + auto inputName = session->GetInputNameAllocated(i, allocator); + auto inputNameStr = inputName.get(); + + inputInt64Tensor.clear(); + inputInt64Tensor.push_back(inputInt64[i]); + inputFinal.push_back( + getInputValueInt64(session, inputInt64Tensor, i, batchSize)); + } + + for (int i = FEATURE_SIZE_INT64_OPT; + i < FEATURE_SIZE_INT64_OPT + FEATURE_SIZE_STRING_OPT; i++) { + inputStringTensor.clear(); + inputStringTensor.push_back(inputString[i - FEATURE_SIZE_INT64_OPT]); + inputFinal.push_back(getInputValueString(allocator, session, + inputStringTensor, i, batchSize)); + } + + // Get input name from model + std::vector inputNameList; + for (int i = 0; i < inputCount; i++) { + auto inputName = session->GetInputNameAllocated(i, allocator); + auto inputNameStr = inputName.get(); + inputNameList.push_back(inputNameStr); + } + + // Form input tensor(s) + std::vector inputNameStrFinal; + for (int i = 0; i < inputCount; i++) { + inputNameStrFinal.push_back(inputNameList[i].c_str()); + } + + // Run model + int outputCount = session->GetOutputCount(); + std::vector outputNameList; + for (int i = 0; i < outputCount; i++) { + auto outputName = session->GetOutputNameAllocated(i, allocator); + std::string outputNameStr = outputName.get(); + if (!outputNameStr.empty()) { + outputNameList.push_back(outputNameStr); + } else { + std::string outputNameDefault = "Output_" + std::to_string(i); + outputNameList.push_back(outputNameDefault); } + } - //Run the model - int output_count = session->GetOutputCount(); - std::vector outputNameList; - for (int i = 0; i < output_count; i++) { - auto outputName = session->GetOutputNameAllocated(i, allocator); - std::string outputNameStr = outputName.get(); - if(!outputNameStr.empty()) { - outputNameList.push_back(outputNameStr); - } else { - std::string outputNameDefault = "Output_" + std::to_string(i); - outputNameList.push_back(outputNameDefault); - } - } + std::vector outputNameStrFinal; + for (int i = 0; i < outputCount; i++) { + outputNameStrFinal.push_back(outputNameList[i].c_str()); + } - std::vector outputNameStr_final; - for(int i = 0; i < output_count; i++) { - outputNameStr_final.push_back(outputNameList[i].c_str()); - } - - auto outputTensors = - session->Run(Ort::RunOptions{nullptr}, inputNameStr_final.data(), - input_final.data(), input_count, outputNameStr_final.data(), output_count); + auto outputTensors = session->Run( + Ort::RunOptions{nullptr}, inputNameStrFinal.data(), inputFinal.data(), + inputCount, outputNameStrFinal.data(), outputCount); - //Try to get the result & return - float* output_probability = outputTensors[0].GetTensorMutableData(); - Ort::Value map_out = outputTensors[1].GetValue(static_cast(0), allocator); + // Get result and return + int64_t label = 0; + for (int i = 0; i < batchSize; i++) { + int64_t *outputLabel = outputTensors[0].GetTensorMutableData(); + label = *outputLabel; + } - Ort::Value keys_ort = map_out.GetValue(0, allocator); - int64_t* keys_ret = keys_ort.GetTensorMutableData(); - Ort::Value values_ort = map_out.GetValue(1, allocator); - float* values_ret = values_ort.GetTensorMutableData(); - - return *(values_ret + 1); + return label; } -} // namespace boltONNXRunner - +} // namespace compilerONNXRunner diff --git a/aiframe/include/ONNXRunner.h b/aiframe/include/ONNXRunner.h index 54bf341ed185ec7f71e6f2ca85c56518aec2f0de..bc8535f1391446b305033674ef563f4f78b65a76 100644 --- a/aiframe/include/ONNXRunner.h +++ b/aiframe/include/ONNXRunner.h @@ -1,64 +1,220 @@ -#ifndef BOLT_PROFILE_ONNXRUNNER_H -#define BOLT_PROFILE_ONNXRUNNER_H +#ifndef COMPILER_PROFILE_ONNXRUNNER_H +#define COMPILER_PROFILE_ONNXRUNNER_H -#include -#include -#include -#include #include "onnxruntime_c_api.h" #include "onnxruntime_cxx_api.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include extern "C" { -namespace boltONNXRunner { +namespace compilerONNXRunner { + +const char* MODEL_PATH_OPT = "/usr/lib64/AI4C/optimizer.onnx"; +const int FEATURE_SIZE_INT64_OPT = 6; +const int FEATURE_SIZE_STRING_OPT = 11; + class ONNXRunner { public: - explicit ONNXRunner() {} - explicit ONNXRunner(const char* model_path) { - // prepare model and env - session_options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_ENABLE_BASIC); - session = new Ort::Session(env, model_path, session_options); - } - ~ONNXRunner() { - delete session; - } - float runONNXModel(std::vector input_string, std::vector input_int64, std::vector input_float); + explicit ONNXRunner() {} + explicit ONNXRunner(const char *modelPath) { + // Prepare model and env + sessionOptions.SetGraphOptimizationLevel( + GraphOptimizationLevel::ORT_ENABLE_BASIC); + session = new Ort::Session(env, modelPath, sessionOptions); + } + ~ONNXRunner() { delete session; } + std::vector runONNXModel(std::vector inputString, + std::vector inputInt64, + std::vector inputFloat, int batchSize); + int64_t runONNXModelOptimizer(std::vector inputString, + std::vector inputInt64, + std::vector inputFloat, int batchSize); private: - static Ort::Value getInputValueFloat(Ort::Session *session, - std::vector &input, - int inputIdx); - - static Ort::Value getInputValueString(Ort::AllocatorWithDefaultOptions allocator, - Ort::Session *session, - std::vector &input, - int inputIdx); - - static Ort::Value getInputValueInt64(Ort::Session *session, - std::vector &input, - int inputIdx); - - Ort::SessionOptions session_options; - Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "test"}; - Ort::Session *session; + static Ort::Value getInputValueFloat(Ort::Session *session, + std::vector &input, int inputIdx, + int batchSize); + + static Ort::Value + getInputValueString(Ort::AllocatorWithDefaultOptions allocator, + Ort::Session *session, std::vector &input, + int inputIdx, int batchSize); + + static Ort::Value getInputValueInt64(Ort::Session *session, + std::vector &input, + int inputIdx, int batchSize); + + Ort::SessionOptions sessionOptions; + Ort::Env env{ORT_LOGGING_LEVEL_WARNING, "test"}; + Ort::Session *session; }; -extern ONNXRunner* createONNXRunner(const char* model_path) { - return new ONNXRunner(model_path); +extern ONNXRunner *createONNXRunner(const char *modelPath) { + std::ifstream file(modelPath); + if (file.good()) { + return new ONNXRunner(modelPath); + } else { + return nullptr; + } +} + +extern void deleteONNXRunner(ONNXRunner *instance) { + if (instance != nullptr) { + delete instance; + } +} + +extern std::vector runONNXModel(ONNXRunner *instance, + std::vector inputString, + std::vector inputInt64, + std::vector inputFloat, + int batchSize) { + std::vector nullResult; + if (instance != nullptr) { + return instance->runONNXModel(inputString, inputInt64, inputFloat, + batchSize); + } else { + return nullResult; + } +} + +static bool startsWithPatt(const std::string &str, const std::string &pattern) { + return str.rfind(pattern, 0) == 0; } -extern void deleteONNXRunner(ONNXRunner* instance) { - if (instance != nullptr) { - delete instance; +static bool optionComparator(const std::string &str1, const std::string &str2) { + for (size_t i = 0; i < str1.size() && i < str2.size(); ++i) { + char c1 = str1[i]; + char c2 = str2[i]; + if (std::isupper(c1) && !std::isupper(c2)) { + return 1; + } else if (!std::isupper(c1) && std::isupper(c2)) { + return 0; + } else if (c1 != c2) { + return c1 > c2; } + } + + return str1.size() < str2.size(); +} + +static void truncatePrefix(const std::string &str, std::string &strPrefix) { + const char prefixIndicator = '_'; + size_t idx = str.find(prefixIndicator); + if (idx == std::string::npos) + strPrefix = str; + else + strPrefix = str.substr(0, idx + 1); +} + +static std::string encodeStringFeature(const std::string &str) { + unsigned char hash[SHA256_DIGEST_LENGTH]; + SHA256_CTX sha256; + SHA256_Init(&sha256); + SHA256_Update(&sha256, str.c_str(), str.size()); + SHA256_Final(hash, &sha256); + + std::stringstream ss; + for (int i = 0; i < SHA256_DIGEST_LENGTH; i++) { + ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash[i]; + } + return ss.str(); } -extern float runONNXModel(ONNXRunner* instance, std::vector input_string, std::vector input_int64, std::vector input_float) { - if (instance != nullptr) { - return instance->runONNXModel(input_string, input_int64, input_float); - } else { - return -1; +static void preprocessData(std::vector &inputString, + std::vector &inputInt64, int argcSW, + const char **argvSW, const char *mcpuOption, + int argcHW, int64_t *argvHW) { + const char *outputOption = "-o"; + const char *macroPrefix = "-D"; + const char *needle = "--param"; + const char *flagPrefix = "-"; + const char *defaultOption = "-fdefault-option"; + const int defaultIntVal = 0; + + // Preprocessing string features. + std::string outputFile = ""; + std::vector inputStringRaw = {std::string(mcpuOption)}; + std::vector macroOptions; + for (int i = 0; i < argcSW; ++i) { + std::string opt = std::string(argvSW[i]); + if (startsWithPatt(opt, macroPrefix)) { + macroOptions.push_back(opt); + } + if (i + 1 < argcSW && opt.compare(outputOption) == 0) { + truncatePrefix(std::string(argvSW[i + 1]), outputFile); + } + } + inputStringRaw.push_back(outputFile); + + std::sort(macroOptions.begin(), macroOptions.end(), optionComparator); + for (size_t i = 0; i < macroOptions.size() && + (int)inputStringRaw.size() < FEATURE_SIZE_STRING_OPT; + ++i) { + inputStringRaw.push_back(macroOptions[i]); + } + + for (int i = 0; + i < argcSW && (int)inputStringRaw.size() < FEATURE_SIZE_STRING_OPT; + ++i) { + std::string opt = std::string(argvSW[i]); + if (!startsWithPatt(opt, macroPrefix) && !startsWithPatt(opt, needle) && + startsWithPatt(opt, flagPrefix)) { + inputStringRaw.push_back(opt); } + } + + for (int i = (int)inputStringRaw.size(); i < FEATURE_SIZE_STRING_OPT; ++i) { + inputStringRaw.push_back(defaultOption); + } + for (size_t i = 0; i < inputStringRaw.size(); ++i) { + inputString.push_back(encodeStringFeature(inputStringRaw[i])); + } + + // Preprocessing int64 features. + for (int i = 0; i < argcHW && i < FEATURE_SIZE_INT64_OPT; ++i) { + inputInt64.push_back(argvHW[i]); + } + + for (int i = (int)inputInt64.size(); i < FEATURE_SIZE_INT64_OPT; ++i) { + inputInt64.push_back(defaultIntVal); + } +} + +extern int64_t runONNXModelOptimizer(int argcSW, const char **argvSW, + const char *mcpuOption, int argcHW, + int64_t *argvHW) { + // Create model runner. + ONNXRunner *instance = createONNXRunner(MODEL_PATH_OPT); + if (instance == nullptr) { + return -1; + } + + // Preprocess data. + std::vector inputString; + std::vector inputInt64; + std::vector inputFloat; + preprocessData(inputString, inputInt64, argcSW, argvSW, mcpuOption, argcHW, + argvHW); + + // Run model. + int64_t output; + if (instance != nullptr) { + output = + instance->runONNXModelOptimizer(inputString, inputInt64, inputFloat, 1); + } + + // Delete model runner. + deleteONNXRunner(instance); + return output; } -} // namespace boltONNXRunner +} // namespace compilerONNXRunner } // extern "C" #endif diff --git a/models/optimizer.onnx b/models/optimizer.onnx new file mode 100644 index 0000000000000000000000000000000000000000..8218b0c2f589a4fd6d9010e67f4e23dab6358f0e Binary files /dev/null and b/models/optimizer.onnx differ