From 0bd4107a2b7de625830ad6e4ede76b35d920b814 Mon Sep 17 00:00:00 2001 From: Sizov Nikita Date: Thu, 2 Nov 2023 22:58:49 +0300 Subject: [PATCH] Fix low/high surrogate character dump Issue: https://gitee.com/openharmony/arkcompiler_ets_frontend/issues/I8I1SX?from=project-issue Testing: gtest Signed-off-by: Sizov Nikita --- ets2panda/checker/ASchecker.cpp | 5 + ets2panda/checker/ETSchecker.cpp | 5 + ets2panda/checker/JSchecker.cpp | 5 + ets2panda/checker/TSchecker.cpp | 5 + ets2panda/es2panda.h | 1 + ets2panda/parser/program/program.cpp | 6 + ets2panda/parser/program/program.h | 2 + ets2panda/test/CMakeLists.txt | 16 ++- ets2panda/test/unit/ast_dumper_test.cpp | 122 ++++++++++++++++++ .../{ => unit}/public/ast_verifier_test.cpp | 0 .../public/es2panda_public_test.cpp | 0 ets2panda/util/helpers.cpp | 7 +- ets2panda/util/options.cpp | 4 + 13 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 ets2panda/test/unit/ast_dumper_test.cpp rename ets2panda/test/{ => unit}/public/ast_verifier_test.cpp (100%) rename ets2panda/test/{ => unit}/public/es2panda_public_test.cpp (100%) diff --git a/ets2panda/checker/ASchecker.cpp b/ets2panda/checker/ASchecker.cpp index 3f096b5ce2..36b7a658c5 100644 --- a/ets2panda/checker/ASchecker.cpp +++ b/ets2panda/checker/ASchecker.cpp @@ -26,6 +26,11 @@ bool ASChecker::StartChecker([[maybe_unused]] varbinder::VarBinder *varbinder, c std::cout << Program()->Dump() << std::endl; } + if (options.op_dump_ast_only_silent) { + Program()->DumpSilent(); + return false; + } + return false; } diff --git a/ets2panda/checker/ETSchecker.cpp b/ets2panda/checker/ETSchecker.cpp index 4d5cfde2fe..4144da1ef3 100644 --- a/ets2panda/checker/ETSchecker.cpp +++ b/ets2panda/checker/ETSchecker.cpp @@ -77,6 +77,11 @@ bool ETSChecker::StartChecker([[maybe_unused]] varbinder::VarBinder *varbinder, std::cout << Program()->Dump() << std::endl; } + if (options.op_dump_ast_only_silent) { + Program()->DumpSilent(); + return false; + } + if (options.parse_only) { return false; } diff --git a/ets2panda/checker/JSchecker.cpp b/ets2panda/checker/JSchecker.cpp index 5b72fec1c2..a844137c2b 100644 --- a/ets2panda/checker/JSchecker.cpp +++ b/ets2panda/checker/JSchecker.cpp @@ -29,6 +29,11 @@ bool JSChecker::StartChecker([[maybe_unused]] varbinder::VarBinder *varbinder, c std::cout << Program()->Dump() << std::endl; } + if (options.op_dump_ast_only_silent) { + Program()->DumpSilent(); + return false; + } + return !options.parse_only; } diff --git a/ets2panda/checker/TSchecker.cpp b/ets2panda/checker/TSchecker.cpp index 6a9eb3977e..2a910c8424 100644 --- a/ets2panda/checker/TSchecker.cpp +++ b/ets2panda/checker/TSchecker.cpp @@ -29,6 +29,11 @@ bool TSChecker::StartChecker([[maybe_unused]] varbinder::VarBinder *varbinder, c std::cout << Program()->Dump() << std::endl; } + if (options.op_dump_ast_only_silent) { + Program()->DumpSilent(); + return false; + } + if (options.parse_only) { return false; } diff --git a/ets2panda/es2panda.h b/ets2panda/es2panda.h index 40774251e5..bd23a328c2 100644 --- a/ets2panda/es2panda.h +++ b/ets2panda/es2panda.h @@ -93,6 +93,7 @@ struct CompilerOptions { bool is_direct_eval {}; bool is_function_eval {}; bool dump_ast {}; + bool op_dump_ast_only_silent {}; bool dump_checked_ast {}; bool dump_asm {}; bool dump_debug_info {}; diff --git a/ets2panda/parser/program/program.cpp b/ets2panda/parser/program/program.cpp index 1ab75d6854..f0adf76b96 100644 --- a/ets2panda/parser/program/program.cpp +++ b/ets2panda/parser/program/program.cpp @@ -29,6 +29,12 @@ std::string Program::Dump() const return dumper.Str(); } +void Program::DumpSilent() const +{ + [[maybe_unused]] ir::AstDumper dumper {ast_, SourceCode()}; + ASSERT(!dumper.Str().empty()); +} + util::StringView Program::PackageClassName(util::StringView class_name) { if (package_name_.Empty()) { diff --git a/ets2panda/parser/program/program.h b/ets2panda/parser/program/program.h index c133118894..88b3db982c 100644 --- a/ets2panda/parser/program/program.h +++ b/ets2panda/parser/program/program.h @@ -217,6 +217,8 @@ public: std::string Dump() const; + void DumpSilent() const; + private: ArenaAllocator *allocator_ {}; varbinder::VarBinder *varbinder_ {}; diff --git a/ets2panda/test/CMakeLists.txt b/ets2panda/test/CMakeLists.txt index ff14df7c4a..a602e480e5 100644 --- a/ets2panda/test/CMakeLists.txt +++ b/ets2panda/test/CMakeLists.txt @@ -78,7 +78,7 @@ if(PANDA_WITH_ETS) panda_add_gtest( NAME es2panda_public_tests SOURCES - public/es2panda_public_test.cpp + unit/public/es2panda_public_test.cpp LIBRARIES es2panda-public es2panda-lib arkassembler arkbytecodeopt INCLUDE_DIRS @@ -95,7 +95,7 @@ if(PANDA_WITH_ETS) panda_add_gtest( NAME es2panda_astverifier_tests SOURCES - public/ast_verifier_test.cpp + unit/public/ast_verifier_test.cpp LIBRARIES es2panda-lib INCLUDE_DIRS @@ -104,5 +104,17 @@ if(PANDA_WITH_ETS) ${PANDA_SANITIZERS_LIST} ) + panda_add_gtest( + NAME es2panda_astdumper_tests + SOURCES + unit/ast_dumper_test.cpp + LIBRARIES + es2panda-public es2panda-lib arkassembler arkbytecodeopt + INCLUDE_DIRS + ${ES2PANDA_PATH} + SANITIZERS + ${PANDA_SANITIZERS_LIST} + ) + add_subdirectory(tsconfig) endif() diff --git a/ets2panda/test/unit/ast_dumper_test.cpp b/ets2panda/test/unit/ast_dumper_test.cpp new file mode 100644 index 0000000000..7dd6eefb4b --- /dev/null +++ b/ets2panda/test/unit/ast_dumper_test.cpp @@ -0,0 +1,122 @@ +/** + * Copyright (c) 2021-2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "macros.h" + +#include "assembler/assembly-program.h" +#include "compiler/core/ASTVerifier.h" +#include "ir/astDump.h" +#include "ir/expressions/literals/stringLiteral.h" + +#include "bytecode_optimizer/bytecodeopt_options.h" +#include "compiler/compiler_logger.h" +#include "mem/arena_allocator.h" +#include "mem/pool_manager.h" +#include "es2panda.h" +#include "util/arktsconfig.h" +#include "util/generateBin.h" +#include "util/options.h" +#include "libpandabase/mem/mem.h" + +class ASTDumperTest : public testing::Test { +public: + ASTDumperTest() + { + constexpr auto COMPILER_SIZE = 268435456; + + panda::mem::MemConfig::Initialize(0, 0, COMPILER_SIZE, 0, 0, 0); + panda::PoolManager::Initialize(panda::PoolType::MMAP); + } + ~ASTDumperTest() override + { + panda::PoolManager::Finalize(); + panda::mem::MemConfig::Finalize(); + }; + + static panda::pandasm::Program *GetProgram(int argc, const char **argv, std::string_view file_name, + std::string_view src) + { + auto options = std::make_unique(); + + if (!options->Parse(argc, argv)) { + std::cerr << options->ErrorMsg() << std::endl; + return nullptr; + } + + panda::Logger::ComponentMask mask {}; + mask.set(panda::Logger::Component::ES2PANDA); + panda::Logger::InitializeStdLogging(panda::Logger::LevelFromString(options->LogLevel()), mask); + + panda::es2panda::Compiler compiler(options->Extension(), options->ThreadCount()); + panda::es2panda::SourceFile input(file_name, src, options->ParseModule()); + + return compiler.Compile(input, options->CompilerOptions()); + } + + NO_COPY_SEMANTIC(ASTDumperTest); + NO_MOVE_SEMANTIC(ASTDumperTest); + +private: +}; + +TEST_F(ASTDumperTest, DumpJsonSimple) +{ + static constexpr std::string_view FILE_NAME = "dummy.ets"; + static constexpr std::string_view SRC = + "\ + function main(args: String[]): int {\ + let a: int = 2;\ + let b: int = 3;\ + return a + b;\ + }"; + + int argc = 0; + const char *argv = ""; + + auto program = std::unique_ptr {GetProgram(argc, &argv, FILE_NAME, SRC)}; + + ASSERT_NE(program, nullptr); + + auto dump_str = program->JsonDump(); + + ASSERT_FALSE(dump_str.empty()); +} + +TEST_F(ASTDumperTest, DumpJsonUTF16Char) +{ + static constexpr std::string_view FILE_NAME = "dummy.ets"; + static constexpr std::string_view SRC = + "\ + function main(args: String[]): int {\ + let a: char = c'\\uDBFF';\ + let b: char = c'\\uDC00';\ + console.log(a);\ + console.log(b);\ + return 0;\ + }"; + + int argc = 0; + const char *argv = ""; + + auto program = std::unique_ptr {GetProgram(argc, &argv, FILE_NAME, SRC)}; + + ASSERT_NE(program, nullptr); + + auto dump_str = program->JsonDump(); + + ASSERT_FALSE(dump_str.empty()); +} diff --git a/ets2panda/test/public/ast_verifier_test.cpp b/ets2panda/test/unit/public/ast_verifier_test.cpp similarity index 100% rename from ets2panda/test/public/ast_verifier_test.cpp rename to ets2panda/test/unit/public/ast_verifier_test.cpp diff --git a/ets2panda/test/public/es2panda_public_test.cpp b/ets2panda/test/unit/public/es2panda_public_test.cpp similarity index 100% rename from ets2panda/test/public/es2panda_public_test.cpp rename to ets2panda/test/unit/public/es2panda_public_test.cpp diff --git a/ets2panda/util/helpers.cpp b/ets2panda/util/helpers.cpp index f0195c7dbc..aafd8d43e7 100644 --- a/ets2panda/util/helpers.cpp +++ b/ets2panda/util/helpers.cpp @@ -42,8 +42,7 @@ #include "ir/ets/etsParameterExpression.h" #include "ir/module/importDeclaration.h" #include "lexer/token/letters.h" -#include -#include +#include "libpandabase/utils/utf.h" namespace panda::es2panda::util { // Helpers @@ -654,8 +653,8 @@ std::string Helpers::CreateEscapedString(const std::string &str) std::string Helpers::UTF16toUTF8(const char16_t c) { - std::wstring_convert, char16_t> convert {}; - return convert.to_bytes(c); + const utf::Utf8Char utf8_ch = utf::ConvertUtf16ToUtf8(c, 0, false); + return std::string(reinterpret_cast(utf8_ch.ch.data()), utf8_ch.n); } std::pair Helpers::SplitSignature(std::string_view signature) diff --git a/ets2panda/util/options.cpp b/ets2panda/util/options.cpp index f5d6b87d98..c7f2350ca8 100644 --- a/ets2panda/util/options.cpp +++ b/ets2panda/util/options.cpp @@ -66,6 +66,8 @@ bool Options::Parse(int argc, const char **argv) panda::PandArg op_module("module", false, "Parse the input as module (JS only option)"); panda::PandArg op_parse_only("parse-only", false, "Parse the input only"); panda::PandArg op_dump_ast("dump-ast", false, "Dump the parsed AST"); + panda::PandArg op_dump_ast_only_silent( + "dump-ast-only-silent", false, "Dump parsed AST with all dumpers available but don't print to stdout"); panda::PandArg op_dump_checked_ast("dump-dynamic-ast", false, "Dump AST with synthetic nodes for dynamic languages"); panda::PandArg op_list_files("list-files", false, "Print names of files that are part of compilation"); @@ -98,6 +100,7 @@ bool Options::Parse(int argc, const char **argv) argparser_->Add(&op_help); argparser_->Add(&op_module); argparser_->Add(&op_dump_ast); + argparser_->Add(&op_dump_ast_only_silent); argparser_->Add(&op_dump_checked_ast); argparser_->Add(&op_parse_only); argparser_->Add(&op_dump_assembly); @@ -290,6 +293,7 @@ bool Options::Parse(int argc, const char **argv) compiler_options_.ts_decl_out = op_ts_decl_out.GetValue(); compiler_options_.dump_asm = op_dump_assembly.GetValue(); compiler_options_.dump_ast = op_dump_ast.GetValue(); + compiler_options_.op_dump_ast_only_silent = op_dump_ast_only_silent.GetValue(); compiler_options_.dump_checked_ast = op_dump_checked_ast.GetValue(); compiler_options_.dump_debug_info = op_dump_debug_info.GetValue(); compiler_options_.is_debug = op_debug_info.GetValue(); -- Gitee