From 0fa5bfe4e14478c9812d49bc8c767c7e4cecf0af Mon Sep 17 00:00:00 2001 From: huyunhui Date: Tue, 1 Aug 2023 12:01:52 +0800 Subject: [PATCH] Deveco support utf-8 in api9 Issue: https://gitee.com/openharmony/arkcompiler_ets_frontend/issues/I7PJZP Signed-off-by: huyunhui --- es2panda/util/helpers.cpp | 30 ++++++++++++++++++++- es2panda/util/helpers.h | 18 ++++++++++++- merge_abc/BUILD.gn | 1 + merge_abc/src/protobufSnapshotGenerator.cpp | 14 +++++++--- ts2panda/ts2abc/BUILD.gn | 1 + 5 files changed, 58 insertions(+), 6 deletions(-) diff --git a/es2panda/util/helpers.cpp b/es2panda/util/helpers.cpp index 49f5abe35d..0d0fc8e0e8 100644 --- a/es2panda/util/helpers.cpp +++ b/es2panda/util/helpers.cpp @@ -476,7 +476,8 @@ void Helpers::OptimizeProgram(panda::pandasm::Program *prog, const std::string bool Helpers::ReadFileToBuffer(const std::string &file, std::stringstream &ss) { - std::ifstream inputStream(panda::os::file::File::GetExtendedFilePath(file), std::ios::binary); + std::ifstream inputStream = Helpers::FileStream( + panda::os::file::File::GetExtendedFilePath(file), std::ios::binary); if (inputStream.fail()) { std::cerr << "Failed to read file to buffer: " << file << std::endl; return false; @@ -485,4 +486,31 @@ bool Helpers::ReadFileToBuffer(const std::string &file, std::stringstream &ss) return true; } +#ifdef PANDA_TARGET_WINDOWS +std::wstring Helpers::Utf8ToUtf16(const std::string &utf8) +{ + std::wstring utf16; + if (utf8.empty()) { + return utf16; + } + + if (utf8.length() > static_cast(std::numeric_limits::max())) { + std::cerr << "Length of filename: " << utf8 << " is too long" << std::endl; + return utf16; + } + + const int utf8Length = static_cast(utf8.length()); + constexpr DWORD kFlags = MB_ERR_INVALID_CHARS; + const int utf16Length = MultiByteToWideChar(CP_UTF8, kFlags, utf8.data(), utf8Length, nullptr, 0); + if (utf16Length == 0) { + std::cerr << "The filename: " << utf8 << " is not a valid utf8 encoding string" << std::endl; + return utf16; + } + + utf16.resize(utf16Length); + MultiByteToWideChar(CP_UTF8, kFlags, utf8.data(), utf8Length, &utf16[0], utf16Length); + return utf16; +} +#endif + } // namespace panda::es2panda::util diff --git a/es2panda/util/helpers.h b/es2panda/util/helpers.h index fe2b222a11..0676f35168 100644 --- a/es2panda/util/helpers.h +++ b/es2panda/util/helpers.h @@ -82,7 +82,10 @@ public: template static T BaseName(T const &path, T const &delims = std::string(panda::os::file::File::GetPathDelim())); static bool ReadFileToBuffer(const std::string &file, std::stringstream &ss); - + static std::wstring Utf8ToUtf16(const std::string &utf8); + template + static T FileStream(const std::string &str, Args &&...args); + static const uint32_t INVALID_INDEX = 4294967295L; static const uint32_t MAX_INT32 = 2147483647; static const uint32_t MAX_INT16 = std::numeric_limits::max(); @@ -109,6 +112,19 @@ T Helpers::BaseName(T const &path, T const &delims) return path.substr(path.find_last_of(delims) + 1); } +template +T Helpers::FileStream(const std::string &str, Args &&...args) +{ + T fileStream; +#ifdef PANDA_TARGET_WINDOWS + std::wstring filename = Helpers::Utf8ToUtf16(str); +#else //for linux and mac + std::string filename = str; +#endif + fileStream.open(filename.c_str(), args...); + return fileStream; +} + } // namespace panda::es2panda::util #endif diff --git a/merge_abc/BUILD.gn b/merge_abc/BUILD.gn index fc97061096..d7dff84c8a 100644 --- a/merge_abc/BUILD.gn +++ b/merge_abc/BUILD.gn @@ -175,6 +175,7 @@ ohos_executable("merge_abc") { ":panda_assembly_proto_static", "$ark_root/libziparchive:libarkziparchive_frontend_static", "$ark_third_party_root/icu/icu4c:static_icuuc", + "../es2panda:es2panda_lib", ] ldflags = [] diff --git a/merge_abc/src/protobufSnapshotGenerator.cpp b/merge_abc/src/protobufSnapshotGenerator.cpp index 8d18f4b45f..f377d501be 100644 --- a/merge_abc/src/protobufSnapshotGenerator.cpp +++ b/merge_abc/src/protobufSnapshotGenerator.cpp @@ -16,6 +16,7 @@ #include "assemblyProgramProto.h" #include "assembly-program.h" #include "protobufSnapshotGenerator.h" +#include "util/helpers.h" namespace panda::proto { void ProtobufSnapshotGenerator::GenerateSnapshot(const panda::pandasm::Program &program, const std::string &outputName) @@ -24,7 +25,8 @@ void ProtobufSnapshotGenerator::GenerateSnapshot(const panda::pandasm::Program & Program::Serialize(program, protoProgram); - std::fstream output(panda::os::file::File::GetExtendedFilePath(outputName), + std::fstream output = panda::es2panda::util::Helpers::FileStream( + panda::os::file::File::GetExtendedFilePath(outputName), std::ios::out | std::ios::trunc | std::ios::binary); if (!output) { std::cerr << "Failed to create: " << outputName << std::endl; @@ -37,7 +39,9 @@ void ProtobufSnapshotGenerator::GenerateSnapshot(const panda::pandasm::Program & void ProtobufSnapshotGenerator::GenerateProgram(const std::string &inputName, panda::pandasm::Program &prog, panda::ArenaAllocator *allocator) { - std::fstream input(panda::os::file::File::GetExtendedFilePath(inputName), std::ios::in | std::ios::binary); + std::fstream input = panda::es2panda::util::Helpers::FileStream( + panda::os::file::File::GetExtendedFilePath(inputName), + std::ios::in | std::ios::binary); if (!input) { std::cerr << "Failed to open: " << inputName << std::endl; return; @@ -53,7 +57,8 @@ void ProtobufSnapshotGenerator::GenerateProgram(const std::string &inputName, pa panda::es2panda::util::ProgramCache *ProtobufSnapshotGenerator::GetCacheContext(const std::string &cacheFilePath, panda::ArenaAllocator *allocator) { - std::fstream input(panda::os::file::File::GetExtendedFilePath(cacheFilePath), + std::fstream input = panda::es2panda::util::Helpers::FileStream( + panda::os::file::File::GetExtendedFilePath(cacheFilePath), std::ios::in | std::ios::binary); if (!input) { return nullptr; @@ -81,7 +86,8 @@ void ProtobufSnapshotGenerator::UpdateCacheFile(const panda::es2panda::util::Pro auto *protoProgram = protoCache.mutable_program(); Program::Serialize(programCache->program, *protoProgram); - std::fstream output(panda::os::file::File::GetExtendedFilePath(cacheFilePath), + std::fstream output = panda::es2panda::util::Helpers::FileStream( + panda::os::file::File::GetExtendedFilePath(cacheFilePath), std::ios::out | std::ios::trunc | std::ios::binary); if (!output) { std::cerr << "Failed to create cache file: " << cacheFilePath << std::endl; diff --git a/ts2panda/ts2abc/BUILD.gn b/ts2panda/ts2abc/BUILD.gn index 19f7bc1a7b..35871e54ad 100755 --- a/ts2panda/ts2abc/BUILD.gn +++ b/ts2panda/ts2abc/BUILD.gn @@ -86,6 +86,7 @@ if (!ark_standalone_build) { ] deps = [ + "../../es2panda:es2panda_lib", "//arkcompiler/ets_frontend/merge_abc:panda_assembly_proto_static", sdk_libc_secshared_dep, ] -- Gitee