From d74787782404f0c5e127530641c00b10d7f68998 Mon Sep 17 00:00:00 2001 From: jiangkaiwen Date: Sat, 7 Jan 2023 22:31:08 +0800 Subject: [PATCH] Es2abc supports new features: rounded space, NextLine, shebang 1.Es2abc supports rounded space 2.Es2abc supports NextLine 3.Es2abc supports shebang Issue:I69DR6 Signed-off-by: jiangkaiwen Change-Id: I1936ca847190994f98af127481deca1a1da2fd77 --- es2panda/lexer/lexer.cpp | 109 ++++++++++++++++++++----- es2panda/lexer/lexer.h | 2 + es2panda/lexer/token/letters.h | 7 ++ es2panda/test/test_tsc_ignore_list.txt | 6 -- 4 files changed, 98 insertions(+), 26 deletions(-) diff --git a/es2panda/lexer/lexer.cpp b/es2panda/lexer/lexer.cpp index 5e80b9718e..7ce6362b09 100644 --- a/es2panda/lexer/lexer.cpp +++ b/es2panda/lexer/lexer.cpp @@ -1227,12 +1227,98 @@ void Lexer::CheckFutureReservedKeyword(TokenType keywordType) ThrowError("Unexpected strict mode reserved keyword"); } +bool Lexer::SkipWhiteSpacesBackwardDefault(uint32_t cp) +{ + Iterator().Backward(1); + cp = Iterator().Peek(); + + if (!SkipWhiteSpacesDefault(cp)) { + return true; + } + + return false; +} + +bool Lexer::SkipWhiteSpacesDefault(uint32_t cp) +{ + if (cp < LEX_ASCII_MAX_BITS) { + return false; + } + + size_t cpSize {}; + cp = Iterator().PeekCp(&cpSize); + + switch (cp) { + case LEX_CHAR_LS: + case LEX_CHAR_PS: { + pos_.nextTokenLine++; + [[fallthrough]]; + } + case LEX_CHAR_NBSP: + case LEX_CHAR_ZWNBSP: { + Iterator().Forward(cpSize); + return true; + } + default: { + return false; + } + } +} + void Lexer::SkipWhiteSpaces() { while (true) { auto cp = Iterator().Peek(); switch (cp) { + case LEX_CHAR_NEXT_LINE_APART1: { + Iterator().Forward(1); + cp = Iterator().Peek(); + if (cp == LEX_CHAR_NEXT_LINE_APART2) { + Iterator().Forward(1); + cp = Iterator().Peek(); + continue; + } else { + if (SkipWhiteSpacesBackwardDefault(cp)){ + return; + } + } + break; + } + case LEX_CHAR_ROUNDED_SP_APART1: { + Iterator().Forward(1); + cp = Iterator().Peek(); + if (cp == LEX_CHAR_ROUNDED_SP_APART2) { + Iterator().Forward(1); + cp = Iterator().Peek(); + if (cp == LEX_CHAR_ROUNDED_SP_APART3) { + Iterator().Forward(1); + cp = Iterator().Peek(); + continue; + } else { + if (SkipWhiteSpacesBackwardDefault(cp)){ + return; + } + } + } else { + if (SkipWhiteSpacesBackwardDefault(cp)){ + return; + } + } + break; + } + case LEX_CHAR_HASH_MARK: { + Iterator().Forward(1); + cp = Iterator().Peek(); + if (cp == LEX_CHAR_EXCLAMATION) { + Iterator().Forward(1); + SkipSingleLineComment(); + continue; + } + + Iterator().Backward(1); + return; + } case LEX_CHAR_CR: { Iterator().Forward(1); @@ -1272,28 +1358,11 @@ void Lexer::SkipWhiteSpaces() return; } default: { - if (cp < LEX_ASCII_MAX_BITS) { + if (SkipWhiteSpacesDefault(cp)) { + continue; + } else { return; } - - size_t cpSize {}; - cp = Iterator().PeekCp(&cpSize); - - switch (cp) { - case LEX_CHAR_LS: - case LEX_CHAR_PS: { - pos_.nextTokenLine++; - [[fallthrough]]; - } - case LEX_CHAR_NBSP: - case LEX_CHAR_ZWNBSP: { - Iterator().Forward(cpSize); - continue; - } - default: { - return; - } - } } } } diff --git a/es2panda/lexer/lexer.h b/es2panda/lexer/lexer.h index b830227025..9303f37e83 100644 --- a/es2panda/lexer/lexer.h +++ b/es2panda/lexer/lexer.h @@ -166,6 +166,8 @@ private: inline static bool IsHexDigit(char32_t ch); inline static bool IsBinaryDigit(char32_t ch); inline static bool IsOctalDigit(char32_t ch); + bool SkipWhiteSpacesDefault(uint32_t cp); + bool SkipWhiteSpacesBackwardDefault(uint32_t cp); friend class KeywordsUtil; friend class TemplateLiteralParserContext; diff --git a/es2panda/lexer/token/letters.h b/es2panda/lexer/token/letters.h index 5ca3771406..a4b55090f4 100644 --- a/es2panda/lexer/token/letters.h +++ b/es2panda/lexer/token/letters.h @@ -144,6 +144,13 @@ namespace panda::es2panda::lexer { #define LEX_VERTICAL_TILDE 0x2E2F /* vertical tilde */ +#define LEX_CHAR_ROUNDED_SP_APART1 0xffffffe3 /* rounded space is 0xe38080 */ +#define LEX_CHAR_ROUNDED_SP_APART2 0xffffff80 +#define LEX_CHAR_ROUNDED_SP_APART3 0xffffff80 + +#define LEX_CHAR_NEXT_LINE_APART1 0xffffffc2 /* nextline is c285 */ +#define LEX_CHAR_NEXT_LINE_APART2 0xffffff85 + #define LEX_TO_ASCII_LOWERCASE(character) ((character) | LEX_CHAR_SP) #define LEX_UTF8_2_BYTE_CODE_POINT_MIN (0x80) diff --git a/es2panda/test/test_tsc_ignore_list.txt b/es2panda/test/test_tsc_ignore_list.txt index 761f6a12a6..91c8e1cc50 100644 --- a/es2panda/test/test_tsc_ignore_list.txt +++ b/es2panda/test/test_tsc_ignore_list.txt @@ -6,9 +6,6 @@ es2panda/test/TypeScript/tests/cases/compiler/collisionArgumentsInterfaceMembers es2panda/test/TypeScript/tests/cases/compiler/computedPropertiesTransformedInOtherwiseNonTSClasses.ts es2panda/test/TypeScript/tests/cases/compiler/constructorOverloads5.ts es2panda/test/TypeScript/tests/cases/compiler/elidedEmbeddedStatementsReplacedWithSemicolon.ts -es2panda/test/TypeScript/tests/cases/compiler/emitBundleWithShebang1.ts -es2panda/test/TypeScript/tests/cases/compiler/fileWithNextLine2.ts -es2panda/test/TypeScript/tests/cases/compiler/genericRecursiveImplicitConstructorErrors2.ts es2panda/test/TypeScript/tests/cases/compiler/globalIsContextualKeyword.ts es2panda/test/TypeScript/tests/cases/compiler/inferenceErasedSignatures.ts es2panda/test/TypeScript/tests/cases/compiler/isLiteral1.ts @@ -19,9 +16,7 @@ es2panda/test/TypeScript/tests/cases/compiler/letInVarDeclOfForIn_ES6.ts es2panda/test/TypeScript/tests/cases/compiler/letInVarDeclOfForOf_ES5.ts es2panda/test/TypeScript/tests/cases/compiler/letInVarDeclOfForOf_ES6.ts es2panda/test/TypeScript/tests/cases/compiler/parameterInitializerBeforeDestructuringEmit.ts -es2panda/test/TypeScript/tests/cases/compiler/shebang.ts es2panda/test/TypeScript/tests/cases/compiler/sigantureIsSubTypeIfTheyAreIdentical.ts -es2panda/test/TypeScript/tests/cases/compiler/sourceMap-LineBreaks.ts es2panda/test/TypeScript/tests/cases/compiler/withStatementInternalComments.ts es2panda/test/TypeScript/tests/cases/conformance/es6/arrowFunction/emitArrowFunctionWhenUsingArguments03_ES6.ts es2panda/test/TypeScript/tests/cases/conformance/es6/arrowFunction/emitArrowFunctionWhenUsingArguments04_ES6.ts @@ -62,7 +57,6 @@ es2panda/test/TypeScript/tests/cases/conformance/interfaces/interfaceDeclaration es2panda/test/TypeScript/tests/cases/conformance/interfaces/interfaceDeclarations/asiPreventsParsingAsInterface03.ts es2panda/test/TypeScript/tests/cases/conformance/interfaces/interfaceDeclarations/asiPreventsParsingAsInterface04.ts es2panda/test/TypeScript/tests/cases/conformance/parser/ecmascript5/RegressionTests/parser579071.ts -es2panda/test/TypeScript/tests/cases/conformance/parser/ecmascript5/parserUnicodeWhitespaceCharacter1.ts es2panda/test/TypeScript/tests/cases/conformance/scanner/ecmascript3/scannerES3NumericLiteral2.ts es2panda/test/TypeScript/tests/cases/conformance/types/keyof/keyofAndIndexedAccess.ts es2panda/test/TypeScript/tests/cases/conformance/types/members/objectTypeWithStringNamedNumericProperty.ts -- Gitee