diff --git a/es2panda/lexer/lexer.cpp b/es2panda/lexer/lexer.cpp index 5e80b9718eeaf3a3db4c7b6a1272ced26a9c2c7e..7ce6362b09113c65dc560273e30e152e7ad7a5d1 100644 --- a/es2panda/lexer/lexer.cpp +++ b/es2panda/lexer/lexer.cpp @@ -1227,12 +1227,98 @@ void Lexer::CheckFutureReservedKeyword(TokenType keywordType) ThrowError("Unexpected strict mode reserved keyword"); } +bool Lexer::SkipWhiteSpacesBackwardDefault(uint32_t cp) +{ + Iterator().Backward(1); + cp = Iterator().Peek(); + + if (!SkipWhiteSpacesDefault(cp)) { + return true; + } + + return false; +} + +bool Lexer::SkipWhiteSpacesDefault(uint32_t cp) +{ + if (cp < LEX_ASCII_MAX_BITS) { + return false; + } + + size_t cpSize {}; + cp = Iterator().PeekCp(&cpSize); + + switch (cp) { + case LEX_CHAR_LS: + case LEX_CHAR_PS: { + pos_.nextTokenLine++; + [[fallthrough]]; + } + case LEX_CHAR_NBSP: + case LEX_CHAR_ZWNBSP: { + Iterator().Forward(cpSize); + return true; + } + default: { + return false; + } + } +} + void Lexer::SkipWhiteSpaces() { while (true) { auto cp = Iterator().Peek(); switch (cp) { + case LEX_CHAR_NEXT_LINE_APART1: { + Iterator().Forward(1); + cp = Iterator().Peek(); + if (cp == LEX_CHAR_NEXT_LINE_APART2) { + Iterator().Forward(1); + cp = Iterator().Peek(); + continue; + } else { + if (SkipWhiteSpacesBackwardDefault(cp)){ + return; + } + } + break; + } + case LEX_CHAR_ROUNDED_SP_APART1: { + Iterator().Forward(1); + cp = Iterator().Peek(); + if (cp == LEX_CHAR_ROUNDED_SP_APART2) { + Iterator().Forward(1); + cp = Iterator().Peek(); + if (cp == LEX_CHAR_ROUNDED_SP_APART3) { + Iterator().Forward(1); + cp = Iterator().Peek(); + continue; + } else { + if (SkipWhiteSpacesBackwardDefault(cp)){ + return; + } + } + } else { + if (SkipWhiteSpacesBackwardDefault(cp)){ + return; + } + } + break; + } + case LEX_CHAR_HASH_MARK: { + Iterator().Forward(1); + cp = Iterator().Peek(); + if (cp == LEX_CHAR_EXCLAMATION) { + Iterator().Forward(1); + SkipSingleLineComment(); + continue; + } + + Iterator().Backward(1); + return; + } case LEX_CHAR_CR: { Iterator().Forward(1); @@ -1272,28 +1358,11 @@ void Lexer::SkipWhiteSpaces() return; } default: { - if (cp < LEX_ASCII_MAX_BITS) { + if (SkipWhiteSpacesDefault(cp)) { + continue; + } else { return; } - - size_t cpSize {}; - cp = Iterator().PeekCp(&cpSize); - - switch (cp) { - case LEX_CHAR_LS: - case LEX_CHAR_PS: { - pos_.nextTokenLine++; - [[fallthrough]]; - } - case LEX_CHAR_NBSP: - case LEX_CHAR_ZWNBSP: { - Iterator().Forward(cpSize); - continue; - } - default: { - return; - } - } } } } diff --git a/es2panda/lexer/lexer.h b/es2panda/lexer/lexer.h index b8302270254e34d7535b8cc14df7819cea242c25..9303f37e83da5d447fd5cad1f75fe2607eda97a2 100644 --- a/es2panda/lexer/lexer.h +++ b/es2panda/lexer/lexer.h @@ -166,6 +166,8 @@ private: inline static bool IsHexDigit(char32_t ch); inline static bool IsBinaryDigit(char32_t ch); inline static bool IsOctalDigit(char32_t ch); + bool SkipWhiteSpacesDefault(uint32_t cp); + bool SkipWhiteSpacesBackwardDefault(uint32_t cp); friend class KeywordsUtil; friend class TemplateLiteralParserContext; diff --git a/es2panda/lexer/token/letters.h b/es2panda/lexer/token/letters.h index 5ca37714066984d42954c382804b8eea8a0d5625..a4b55090f4d7341827cd4eb016e5c9c21a96e2fb 100644 --- a/es2panda/lexer/token/letters.h +++ b/es2panda/lexer/token/letters.h @@ -144,6 +144,13 @@ namespace panda::es2panda::lexer { #define LEX_VERTICAL_TILDE 0x2E2F /* vertical tilde */ +#define LEX_CHAR_ROUNDED_SP_APART1 0xffffffe3 /* rounded space is 0xe38080 */ +#define LEX_CHAR_ROUNDED_SP_APART2 0xffffff80 +#define LEX_CHAR_ROUNDED_SP_APART3 0xffffff80 + +#define LEX_CHAR_NEXT_LINE_APART1 0xffffffc2 /* nextline is c285 */ +#define LEX_CHAR_NEXT_LINE_APART2 0xffffff85 + #define LEX_TO_ASCII_LOWERCASE(character) ((character) | LEX_CHAR_SP) #define LEX_UTF8_2_BYTE_CODE_POINT_MIN (0x80) diff --git a/es2panda/test/test_tsc_ignore_list.txt b/es2panda/test/test_tsc_ignore_list.txt index 761f6a12a60d378dd107004dcfa7ae526836406a..91c8e1cc509c04701c6919c5d962817439fddaa6 100644 --- a/es2panda/test/test_tsc_ignore_list.txt +++ b/es2panda/test/test_tsc_ignore_list.txt @@ -6,9 +6,6 @@ es2panda/test/TypeScript/tests/cases/compiler/collisionArgumentsInterfaceMembers es2panda/test/TypeScript/tests/cases/compiler/computedPropertiesTransformedInOtherwiseNonTSClasses.ts es2panda/test/TypeScript/tests/cases/compiler/constructorOverloads5.ts es2panda/test/TypeScript/tests/cases/compiler/elidedEmbeddedStatementsReplacedWithSemicolon.ts -es2panda/test/TypeScript/tests/cases/compiler/emitBundleWithShebang1.ts -es2panda/test/TypeScript/tests/cases/compiler/fileWithNextLine2.ts -es2panda/test/TypeScript/tests/cases/compiler/genericRecursiveImplicitConstructorErrors2.ts es2panda/test/TypeScript/tests/cases/compiler/globalIsContextualKeyword.ts es2panda/test/TypeScript/tests/cases/compiler/inferenceErasedSignatures.ts es2panda/test/TypeScript/tests/cases/compiler/isLiteral1.ts @@ -19,9 +16,7 @@ es2panda/test/TypeScript/tests/cases/compiler/letInVarDeclOfForIn_ES6.ts es2panda/test/TypeScript/tests/cases/compiler/letInVarDeclOfForOf_ES5.ts es2panda/test/TypeScript/tests/cases/compiler/letInVarDeclOfForOf_ES6.ts es2panda/test/TypeScript/tests/cases/compiler/parameterInitializerBeforeDestructuringEmit.ts -es2panda/test/TypeScript/tests/cases/compiler/shebang.ts es2panda/test/TypeScript/tests/cases/compiler/sigantureIsSubTypeIfTheyAreIdentical.ts -es2panda/test/TypeScript/tests/cases/compiler/sourceMap-LineBreaks.ts es2panda/test/TypeScript/tests/cases/compiler/withStatementInternalComments.ts es2panda/test/TypeScript/tests/cases/conformance/es6/arrowFunction/emitArrowFunctionWhenUsingArguments03_ES6.ts es2panda/test/TypeScript/tests/cases/conformance/es6/arrowFunction/emitArrowFunctionWhenUsingArguments04_ES6.ts @@ -62,7 +57,6 @@ es2panda/test/TypeScript/tests/cases/conformance/interfaces/interfaceDeclaration es2panda/test/TypeScript/tests/cases/conformance/interfaces/interfaceDeclarations/asiPreventsParsingAsInterface03.ts es2panda/test/TypeScript/tests/cases/conformance/interfaces/interfaceDeclarations/asiPreventsParsingAsInterface04.ts es2panda/test/TypeScript/tests/cases/conformance/parser/ecmascript5/RegressionTests/parser579071.ts -es2panda/test/TypeScript/tests/cases/conformance/parser/ecmascript5/parserUnicodeWhitespaceCharacter1.ts es2panda/test/TypeScript/tests/cases/conformance/scanner/ecmascript3/scannerES3NumericLiteral2.ts es2panda/test/TypeScript/tests/cases/conformance/types/keyof/keyofAndIndexedAccess.ts es2panda/test/TypeScript/tests/cases/conformance/types/members/objectTypeWithStringNamedNumericProperty.ts