From 0bf45daa5f1d700c991f39bf951b5b46da15ba49 Mon Sep 17 00:00:00 2001 From: "zha.wei" Date: Wed, 23 Aug 2023 17:22:55 +0800 Subject: [PATCH 1/7] description:ECMA Script 2023 added support for #! comments at the beginning of files Signed-off-by: zha.wei --- es2panda/lexer/lexer.cpp | 10 ++++++++++ es2panda/lexer/lexer.h | 1 + es2panda/lexer/token/sourceLocation.cpp | 16 ++++++++++++++++ es2panda/lexer/token/sourceLocation.h | 1 + 4 files changed, 28 insertions(+) diff --git a/es2panda/lexer/lexer.cpp b/es2panda/lexer/lexer.cpp index 6ba93184a8..8b3222824a 100644 --- a/es2panda/lexer/lexer.cpp +++ b/es2panda/lexer/lexer.cpp @@ -204,6 +204,13 @@ void Lexer::ThrowError(std::string_view message) throw es2panda::Error(es2panda::ErrorType::SYNTAX, message, pos_.line, Iterator().Index()); } +void Lexer::ThrowErrorIndex(std::string_view message, size_t index) +{ + lexer::LineIndex lineIndex = parserContext_->GetProgram()->GetLineIndex(); + SourceLocation pos = lineIndex.GetLocation(index); + throw es2panda::Error(es2panda::ErrorType::SYNTAX, message, pos.line, pos.col); +} + void Lexer::CheckNumberLiteralEnd() { if (Iterator().Peek() == LEX_CHAR_LOWERCASE_N) { @@ -1240,6 +1247,9 @@ void Lexer::SkipWhiteSpaces() Iterator().Forward(1); cp = Iterator().Peek(); if (cp == LEX_CHAR_EXCLAMATION) { + if (Iterator().Index() != 1) { + ThrowErrorIndex("Invalid or unexpected token", Iterator().Index()); + } Iterator().Forward(1); SkipSingleLineComment(); continue; diff --git a/es2panda/lexer/lexer.h b/es2panda/lexer/lexer.h index 8e56da6ed2..5d46e36dd4 100644 --- a/es2panda/lexer/lexer.h +++ b/es2panda/lexer/lexer.h @@ -99,6 +99,7 @@ private: RegExpFlags ScanRegExpFlags(); void ThrowError(std::string_view message); + void ThrowErrorIndex(std::string_view message, size_t index); void SetTokenStart(); void SetTokenEnd(); diff --git a/es2panda/lexer/token/sourceLocation.cpp b/es2panda/lexer/token/sourceLocation.cpp index 8cb80a458b..11477c2e98 100644 --- a/es2panda/lexer/token/sourceLocation.cpp +++ b/es2panda/lexer/token/sourceLocation.cpp @@ -99,4 +99,20 @@ SourceLocation LineIndex::GetLocation(SourcePosition pos) noexcept return SourceLocation(line + 1, col + 1); } +SourceLocation LineIndex::GetLocation(size_t index) noexcept +{ + size_t line = 0; + size_t col = 0; + ASSERT(index < entrys_.back().lineStart); + for (size_t pos = 0; pos < entrys_.size(); ++pos) { + if (index > entrys_[pos].lineStart) { + ++line; + continue; + } + col = index - entrys_[pos - 1].lineStart + 1; + break; + } + + return SourceLocation(line, col); +} } // namespace panda::es2panda::lexer diff --git a/es2panda/lexer/token/sourceLocation.h b/es2panda/lexer/token/sourceLocation.h index a9c193ed16..6e92e9d286 100644 --- a/es2panda/lexer/token/sourceLocation.h +++ b/es2panda/lexer/token/sourceLocation.h @@ -104,6 +104,7 @@ public: ~LineIndex() = default; SourceLocation GetLocation(SourcePosition pos) noexcept; + SourceLocation GetLocation(size_t index) noexcept; private: std::vector entrys_; -- Gitee From a290f362e201fec2693c8a08ebdd51ca840ba314 Mon Sep 17 00:00:00 2001 From: "zha.wei" Date: Fri, 25 Aug 2023 15:59:40 +0800 Subject: [PATCH 2/7] description: add es2panda parser test case Signed-off-by: zha.wei --- .../js/test-hashbang-comment-1-expected.txt | 72 +++++++++++++++++++ .../test/parser/js/test-hashbang-comment-1.js | 18 +++++ .../js/test-hashbang-comment-2-expected.txt | 1 + .../test/parser/js/test-hashbang-comment-2.js | 18 +++++ .../js/test-hashbang-comment-3-expected.txt | 1 + .../test/parser/js/test-hashbang-comment-3.js | 18 +++++ 6 files changed, 128 insertions(+) create mode 100644 es2panda/test/parser/js/test-hashbang-comment-1-expected.txt create mode 100644 es2panda/test/parser/js/test-hashbang-comment-1.js create mode 100644 es2panda/test/parser/js/test-hashbang-comment-2-expected.txt create mode 100644 es2panda/test/parser/js/test-hashbang-comment-2.js create mode 100644 es2panda/test/parser/js/test-hashbang-comment-3-expected.txt create mode 100644 es2panda/test/parser/js/test-hashbang-comment-3.js diff --git a/es2panda/test/parser/js/test-hashbang-comment-1-expected.txt b/es2panda/test/parser/js/test-hashbang-comment-1-expected.txt new file mode 100644 index 0000000000..0e9a0690b0 --- /dev/null +++ b/es2panda/test/parser/js/test-hashbang-comment-1-expected.txt @@ -0,0 +1,72 @@ +{ + "type": "Program", + "statements": [ + { + "type": "ExpressionStatement", + "expression": { + "type": "CallExpression", + "callee": { + "type": "Identifier", + "name": "print", + "loc": { + "start": { + "line": 18, + "column": 1 + }, + "end": { + "line": 18, + "column": 6 + } + } + }, + "arguments": [ + { + "type": "StringLiteral", + "value": "hello world", + "loc": { + "start": { + "line": 18, + "column": 7 + }, + "end": { + "line": 18, + "column": 20 + } + } + } + ], + "optional": false, + "loc": { + "start": { + "line": 18, + "column": 1 + }, + "end": { + "line": 18, + "column": 21 + } + } + }, + "loc": { + "start": { + "line": 18, + "column": 1 + }, + "end": { + "line": 18, + "column": 21 + } + } + } + ], + "loc": { + "start": { + "line": 1, + "column": 1 + }, + "end": { + "line": 18, + "column": 21 + } + } +} diff --git a/es2panda/test/parser/js/test-hashbang-comment-1.js b/es2panda/test/parser/js/test-hashbang-comment-1.js new file mode 100644 index 0000000000..68746e0780 --- /dev/null +++ b/es2panda/test/parser/js/test-hashbang-comment-1.js @@ -0,0 +1,18 @@ +#! Copyright (c) 2023 Shenzhen Kaihong Digital Industry Development Co., Ltd. +/** + * Copyright (c) 2023 Shenzhen Kaihong Digital Industry Development Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// slash comment +print('hello world') \ No newline at end of file diff --git a/es2panda/test/parser/js/test-hashbang-comment-2-expected.txt b/es2panda/test/parser/js/test-hashbang-comment-2-expected.txt new file mode 100644 index 0000000000..115339bb1f --- /dev/null +++ b/es2panda/test/parser/js/test-hashbang-comment-2-expected.txt @@ -0,0 +1 @@ +SyntaxError: Invalid or unexpected token [test-hashbang-comment-2.js:16:2] diff --git a/es2panda/test/parser/js/test-hashbang-comment-2.js b/es2panda/test/parser/js/test-hashbang-comment-2.js new file mode 100644 index 0000000000..c035e60ac9 --- /dev/null +++ b/es2panda/test/parser/js/test-hashbang-comment-2.js @@ -0,0 +1,18 @@ +/** + * Copyright (c) 2023 Shenzhen Kaihong Digital Industry Development Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#! Copyright (c) 2023 Shenzhen Kaihong Digital Industry Development Co., Ltd. +// slash comment +print('hello world') \ No newline at end of file diff --git a/es2panda/test/parser/js/test-hashbang-comment-3-expected.txt b/es2panda/test/parser/js/test-hashbang-comment-3-expected.txt new file mode 100644 index 0000000000..03103b85be --- /dev/null +++ b/es2panda/test/parser/js/test-hashbang-comment-3-expected.txt @@ -0,0 +1 @@ +SyntaxError: Invalid or unexpected token [test-hashbang-comment-3.js:18:2] diff --git a/es2panda/test/parser/js/test-hashbang-comment-3.js b/es2panda/test/parser/js/test-hashbang-comment-3.js new file mode 100644 index 0000000000..e91e04fc6c --- /dev/null +++ b/es2panda/test/parser/js/test-hashbang-comment-3.js @@ -0,0 +1,18 @@ +/** + * Copyright (c) 2023 Shenzhen Kaihong Digital Industry Development Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// slash comment +print('hello world') +#! Copyright (c) 2023 Shenzhen Kaihong Digital Industry Development Co., Ltd. \ No newline at end of file -- Gitee From e1cba89396c27da7e6d94ba47eda760d0949a4af Mon Sep 17 00:00:00 2001 From: "zha.wei" Date: Wed, 20 Sep 2023 11:37:25 +0800 Subject: [PATCH 3/7] description:#! linecomments code review fix Signed-off-by: zha.wei --- es2panda/lexer/lexer.cpp | 4 ++++ es2panda/lexer/token/sourceLocation.cpp | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/es2panda/lexer/lexer.cpp b/es2panda/lexer/lexer.cpp index 8b3222824a..b86ad0c50e 100644 --- a/es2panda/lexer/lexer.cpp +++ b/es2panda/lexer/lexer.cpp @@ -1248,6 +1248,10 @@ void Lexer::SkipWhiteSpaces() cp = Iterator().Peek(); if (cp == LEX_CHAR_EXCLAMATION) { if (Iterator().Index() != 1) { + /* + * according to ECMA-262 specification item 12.5 Hashbang Comments are location-sensitive. + * only allowed occurs at the beginning of files, other position is illegal. + */ ThrowErrorIndex("Invalid or unexpected token", Iterator().Index()); } Iterator().Forward(1); diff --git a/es2panda/lexer/token/sourceLocation.cpp b/es2panda/lexer/token/sourceLocation.cpp index 11477c2e98..e50adaf3d3 100644 --- a/es2panda/lexer/token/sourceLocation.cpp +++ b/es2panda/lexer/token/sourceLocation.cpp @@ -103,13 +103,15 @@ SourceLocation LineIndex::GetLocation(size_t index) noexcept { size_t line = 0; size_t col = 0; + size_t prev = 0; ASSERT(index < entrys_.back().lineStart); for (size_t pos = 0; pos < entrys_.size(); ++pos) { if (index > entrys_[pos].lineStart) { + prev = pos; // save prev pos ++line; continue; } - col = index - entrys_[pos - 1].lineStart + 1; + col = index - entrys_[prev].lineStart; break; } -- Gitee From 7ee69e16f94f945ba8d640be26e5051e1b5799a0 Mon Sep 17 00:00:00 2001 From: "zha.wei" Date: Wed, 20 Sep 2023 13:57:14 +0800 Subject: [PATCH 4/7] description:fix hash bang test case Signed-off-by: zha.wei --- es2panda/test/parser/js/test-hashbang-comment-2-expected.txt | 2 +- es2panda/test/parser/js/test-hashbang-comment-3-expected.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/es2panda/test/parser/js/test-hashbang-comment-2-expected.txt b/es2panda/test/parser/js/test-hashbang-comment-2-expected.txt index 115339bb1f..eb5b1a5c25 100644 --- a/es2panda/test/parser/js/test-hashbang-comment-2-expected.txt +++ b/es2panda/test/parser/js/test-hashbang-comment-2-expected.txt @@ -1 +1 @@ -SyntaxError: Invalid or unexpected token [test-hashbang-comment-2.js:16:2] +SyntaxError: Invalid or unexpected token [test-hashbang-comment-2.js:16:1] diff --git a/es2panda/test/parser/js/test-hashbang-comment-3-expected.txt b/es2panda/test/parser/js/test-hashbang-comment-3-expected.txt index 03103b85be..dd7c0a615d 100644 --- a/es2panda/test/parser/js/test-hashbang-comment-3-expected.txt +++ b/es2panda/test/parser/js/test-hashbang-comment-3-expected.txt @@ -1 +1 @@ -SyntaxError: Invalid or unexpected token [test-hashbang-comment-3.js:18:2] +SyntaxError: Invalid or unexpected token [test-hashbang-comment-3.js:18:1] -- Gitee From cd39f07eb21599c4d4e2430c685778fbdd61041f Mon Sep 17 00:00:00 2001 From: "zha.wei" Date: Thu, 21 Sep 2023 16:33:54 +0800 Subject: [PATCH 5/7] description:modify for more readability Signed-off-by: zha.wei --- es2panda/lexer/token/sourceLocation.cpp | 18 +++++++++++------- es2panda/lexer/token/sourceLocation.h | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/es2panda/lexer/token/sourceLocation.cpp b/es2panda/lexer/token/sourceLocation.cpp index e50adaf3d3..1888a8ad52 100644 --- a/es2panda/lexer/token/sourceLocation.cpp +++ b/es2panda/lexer/token/sourceLocation.cpp @@ -40,6 +40,11 @@ void OffsetEntry::AddCol(size_t offset) } } +size_t OffsetEntry::GetOffset() +{ + return offset_; +} + LineIndex::LineIndex(const util::StringView &source) noexcept { auto iter = util::StringView::Iterator(source); @@ -101,18 +106,17 @@ SourceLocation LineIndex::GetLocation(SourcePosition pos) noexcept SourceLocation LineIndex::GetLocation(size_t index) noexcept { - size_t line = 0; - size_t col = 0; - size_t prev = 0; + size_t line = 1; + size_t col = 1; ASSERT(index < entrys_.back().lineStart); for (size_t pos = 0; pos < entrys_.size(); ++pos) { - if (index > entrys_[pos].lineStart) { - prev = pos; // save prev pos + if (index >= entrys_[pos].lineStart && index < entrys_[pos].GetOffset()) { + col = index - entrys_[pos].lineStart; + break; + } else { ++line; continue; } - col = index - entrys_[prev].lineStart; - break; } return SourceLocation(line, col); diff --git a/es2panda/lexer/token/sourceLocation.h b/es2panda/lexer/token/sourceLocation.h index 6e92e9d286..9f1004871a 100644 --- a/es2panda/lexer/token/sourceLocation.h +++ b/es2panda/lexer/token/sourceLocation.h @@ -87,6 +87,7 @@ public: ~OffsetEntry() = default; void AddCol(size_t offset); + size_t GetOffset(); std::vector ranges {}; size_t lineStart {}; -- Gitee From 085885eec39714af33effecbbfeded1349452c32 Mon Sep 17 00:00:00 2001 From: "zha.wei" Date: Thu, 21 Sep 2023 17:05:14 +0800 Subject: [PATCH 6/7] description: add comments Signed-off-by: zha.wei --- es2panda/lexer/token/sourceLocation.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/es2panda/lexer/token/sourceLocation.cpp b/es2panda/lexer/token/sourceLocation.cpp index 1888a8ad52..7f47655694 100644 --- a/es2panda/lexer/token/sourceLocation.cpp +++ b/es2panda/lexer/token/sourceLocation.cpp @@ -108,9 +108,9 @@ SourceLocation LineIndex::GetLocation(size_t index) noexcept { size_t line = 1; size_t col = 1; - ASSERT(index < entrys_.back().lineStart); + ASSERT(index < entrys_.back().lineStart); // EOF for (size_t pos = 0; pos < entrys_.size(); ++pos) { - if (index >= entrys_[pos].lineStart && index < entrys_[pos].GetOffset()) { + if (index >= entrys_[pos].lineStart && index < entrys_[pos].GetOffset()) { // line ends with newline punctuator col = index - entrys_[pos].lineStart; break; } else { -- Gitee From a8b49694b5cae5fd25ddc46acce2ab3fc41b8aaa Mon Sep 17 00:00:00 2001 From: "zha.wei" Date: Sat, 7 Oct 2023 14:59:13 +0800 Subject: [PATCH 7/7] description:fix review issue Signed-off-by: zha.wei --- es2panda/lexer/lexer.cpp | 2 +- es2panda/lexer/token/sourceLocation.cpp | 16 ++++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/es2panda/lexer/lexer.cpp b/es2panda/lexer/lexer.cpp index b86ad0c50e..cab220dbb1 100644 --- a/es2panda/lexer/lexer.cpp +++ b/es2panda/lexer/lexer.cpp @@ -1252,7 +1252,7 @@ void Lexer::SkipWhiteSpaces() * according to ECMA-262 specification item 12.5 Hashbang Comments are location-sensitive. * only allowed occurs at the beginning of files, other position is illegal. */ - ThrowErrorIndex("Invalid or unexpected token", Iterator().Index()); + ThrowErrorIndex("Invalid or unexpected token", Iterator().Index() - 1); } Iterator().Forward(1); SkipSingleLineComment(); diff --git a/es2panda/lexer/token/sourceLocation.cpp b/es2panda/lexer/token/sourceLocation.cpp index 7f47655694..8e13aefe76 100644 --- a/es2panda/lexer/token/sourceLocation.cpp +++ b/es2panda/lexer/token/sourceLocation.cpp @@ -106,19 +106,15 @@ SourceLocation LineIndex::GetLocation(SourcePosition pos) noexcept SourceLocation LineIndex::GetLocation(size_t index) noexcept { - size_t line = 1; - size_t col = 1; + SourcePosition sp; ASSERT(index < entrys_.back().lineStart); // EOF - for (size_t pos = 0; pos < entrys_.size(); ++pos) { - if (index >= entrys_[pos].lineStart && index < entrys_[pos].GetOffset()) { // line ends with newline punctuator - col = index - entrys_[pos].lineStart; + for (size_t line = 0; line < entrys_.size(); ++line) { + if (index >= entrys_[line].lineStart && index < entrys_[line].GetOffset()) { // line ends with newline punctuator + sp.index = index; + sp.line = line; break; - } else { - ++line; - continue; } } - - return SourceLocation(line, col); + return GetLocation(sp); } } // namespace panda::es2panda::lexer -- Gitee