diff --git a/src/MapleFE/.gitignore b/src/MapleFE/.gitignore index 46890e54e45d66060ba85adb278ca197da8524d8..533695ff01a419af9567d36b8d2cf41970296906 100644 --- a/src/MapleFE/.gitignore +++ b/src/MapleFE/.gitignore @@ -10,6 +10,7 @@ output/ *.png *.out.ts *.ts.ast +*.java.ast gdbfile test/typescript/**/*.cpp test/typescript/**/*.h diff --git a/src/MapleFE/scripts/perf-java.sh b/src/MapleFE/scripts/perf-java.sh new file mode 100755 index 0000000000000000000000000000000000000000..b838fd3675cf0b76f7ba0a6bc9aee3d477e59d70 --- /dev/null +++ b/src/MapleFE/scripts/perf-java.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# This script is to measure the runtime performance of java2ast and other executables +# If perf is not installed yet, please install the package linux-tools-common with all dependencies +TS2AST=$(dirname $0)/../output/java/bin/java2ast +CMD="sudo perf record -e cpu-cycles,cache-misses --call-graph fp -F 10000 -o perf.data" +if [ $# -eq 0 ]; then + echo "Usage: $0 " + echo " $0 " + exit 1 +elif [ $# -eq 1 -a "$(basename $1)" != "$(basename $1 .ts)" ]; then + echo $CMD $TS2AST "$@" + $CMD $TS2AST "$@" +else + echo $CMD "$@" + $CMD "$@" +fi +echo sudo perf report +sudo perf report diff --git a/src/MapleFE/shared/include/ast_type.h b/src/MapleFE/shared/include/ast_type.h index 8186eabd80377cdb7754c0dc88c6bde3e496ecb4..0f4afe5ee73d61dff5d91d629ab2d40b51d7d471 100644 --- a/src/MapleFE/shared/include/ast_type.h +++ b/src/MapleFE/shared/include/ast_type.h @@ -103,7 +103,7 @@ public: void SetId(TreeNode *n) {mId = n; SETPARENT(n);} unsigned GetUnionInterTypesNum() {return mUnionInterTypes.GetNum();} - void AddUnionInterType(TreeNode *n); + void AddUnionInterType(TreeNode *n, bool front = false); TreeNode* GetUnionInterType(unsigned i) {return mUnionInterTypes.ValueAtIndex(i);} void SetUnionInterType(unsigned i, TreeNode* n) {*(mUnionInterTypes.RefAtIndex(i)) = n; SETPARENT(n);} diff --git a/src/MapleFE/shared/include/container.h b/src/MapleFE/shared/include/container.h index 39066b4e9a211b7f2ce0b102e307216ce8099dd9..828a8379e00034339b1eeca24fbc55045b9a79f9 100644 --- a/src/MapleFE/shared/include/container.h +++ b/src/MapleFE/shared/include/container.h @@ -1,5 +1,6 @@ /* * Copyright (C) [2020] Futurewei Technologies, Inc. All rights reverved. +* Copyright (C) 2022 Tencent. All rights reverved. * * OpenArkFE is licensed under the Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -33,6 +34,7 @@ #define __CONTAINER_H__ #include +#include #include "mempool.h" #include "massert.h" #include "macros.h" @@ -777,6 +779,338 @@ public: } }; +//////////////////////////////////////////////////////////////////////// +// GuamianFast uses unordered map to store the Knob-s in order to +// speed up the searching with big number of data. +//////////////////////////////////////////////////////////////////////// + +template class GuamianFast { +private: + struct Elem{ + E mData; + Elem *mNext; + }; + + // Sometimes people need save certain additional information to + // each knob. So we define mData. + struct Knob{ + D mData; + Elem *mChildren; // pointing to the first child + }; + + MemPool mMemPool; + std::unordered_map mKnobs; + + // allocate a new knob + Knob* NewKnob() { + Knob *knob = (Knob*)mMemPool.Alloc(sizeof(Knob)); + knob->mData = 0; + knob->mChildren = NULL; + return knob; + } + + // allocate a new element + Elem* NewElem() { + Elem *elem = (Elem*)mMemPool.Alloc(sizeof(Elem)); + elem->mNext = NULL; + elem->mData = 0; + return elem; + } + + // Sometimes people want to have a sequence of operations like, + // Get the knob, + // Add one element, on the knob + // Add more elements, on the knob. + // This is common scenario. To implement, it requires a temporary + // pointer to the located knob. This temp knob is used ONLY when + // paired operations, PairedFindOrCreateKnob() and PairedAddElem() + struct { + Knob *mKnob; + K mKey; + }mTempKnob; + +private: + // Just try to find the Knob. + // return NULL if fails. + Knob* FindKnob(K key) { + Knob *result = NULL; + auto search = mKnobs.find(key); + if (search != mKnobs.end()) + result = search->second; + return result; + } + + // Try to find the Knob. Create one if failed. + Knob* FindOrCreateKnob(K key) { + Knob *knob = FindKnob(key); + if (!knob) { + knob = NewKnob(); + mKnobs.insert(std::make_pair(key, knob)); + } + return knob; + } + + // Add an element to knob. It's the caller's duty to assure + // knob is not NULL. + void AddElem(Knob *knob, E data) { + Elem *elem = knob->mChildren; + Elem *found = NULL; + while (elem) { + if (elem->mData == data) { + found = elem; + break; + } + elem = elem->mNext; + } + + if (!found) { + Elem *e = NewElem(); + e->mData = data; + e->mNext = knob->mChildren; + knob->mChildren = e; + } + } + + // return true : if find the element + // false : if fail + bool FindElem(Knob *knob, E data) { + Elem *elem = knob->mChildren; + while (elem) { + if (elem->mData == data) + return true; + elem = elem->mNext; + } + return false; + } + + // Remove elem from the list. If elem doesn't exist, exit quietly. + // It's caller's duty to assure elem exists. + void RemoveElem(Knob *knob, E data) { + Elem *elem = knob->mChildren; + Elem *elem_prev = NULL; + Elem *target = NULL; + while (elem) { + if (elem->mData == data) { + target = elem; + break; + } + elem_prev = elem; + elem = elem->mNext; + } + + if (target) { + if (target == knob->mChildren) + knob->mChildren = target->mNext; + else + elem_prev->mNext = target->mNext; + } + } + + // Move the element to be the first child of knob. + // It's the caller's duty to make sure 'data' does exist + // in knob's children. + void MoveElemToHead(Knob *knob, E data) { + Elem *target_elem = NULL; + Elem *elem = knob->mChildren; + Elem *elem_prev = NULL; + while (elem) { + if (elem->mData == data) { + target_elem = elem; + break; + } + elem_prev = elem; + elem = elem->mNext; + } + + if (target_elem && (target_elem != knob->mChildren)) { + elem_prev->mNext = target_elem->mNext; + target_elem->mNext = knob->mChildren; + knob->mChildren = target_elem; + } + } + + // Try to find the first child of Knob k. Return the data. + // found is set to false if fails, or true. + // [NOTE] It's the user's responsibilty to make sure the Knob + // of 'key' exists. + E FindFirstElem(Knob *knob, bool &found) { + Elem *e = knob->mChildren; + if (!e) { + found = false; + return 0; + } + found = true; + return e->mData; + } + + // return num of elements in knob. + // It's caller's duty to assure knob is not NULL. + unsigned NumOfElem(Knob *knob) { + Elem *e = knob->mChildren; + unsigned c = 0; + while(e) { + c++; + e = e->mNext; + } + return c; + } + + // Return the idx-th element in knob. + // It's caller's duty to assure the validity of return value. + // It doesn't check validity here. + // Index starts from 0. + E GetElemAtIndex(Knob *knob, unsigned idx) { + Elem *e = knob->mChildren; + unsigned c = 0; + E data; + while(e) { + if (c == idx) { + data = e->mData; + break; + } + c++; + e = e->mNext; + } + return data; + } + +public: + GuamianFast() {mTempKnob.mKnob = NULL;} + ~GuamianFast(){Release();} + + void AddElem(K key, E data) { + Knob *knob = FindOrCreateKnob(key); + AddElem(knob, data); + } + + // If 'data' doesn't exist, it ends quietly + void RemoveElem(K key, E data) { + Knob *knob = FindOrCreateKnob(key); + RemoveElem(knob, data); + } + + // Try to find the first child of Knob k. Return the data. + // found is set to false if fails, or true. + // [NOTE] It's the user's responsibilty to make sure the Knob + // of 'key' exists. + E FindFirstElem(K key, bool &found) { + Knob *knob = FindKnob(key); + if (!knob) { + found = false; + return 0; // return value doesn't matter when fails. + } + E data = FindFirstElem(knob, found); + return data; + } + + // return true : if find the element + // false : if fail + bool FindElem(K key, E data) { + Knob *knob = FindKnob(key); + if (!knob) + return false; + return FindElem(knob, data); + } + + // Move element to be the header + // If 'data' doesn't exist, it ends quietly. + void MoveElemToHead(K key, E data) { + Knob *knob = FindKnob(key); + if (!knob) + return; + MoveElemToHead(knob, data); + } + + ///////////////////////////////////////////////////////// + // Paired operations start with finding a knob. It can + // be either PairedFindKnob() or PairedFindOrCreateKnob() + // Following that, there could be any number of operations + // like searching, adding, moving an element. + ///////////////////////////////////////////////////////// + + void PairedFindOrCreateKnob(K key) { + mTempKnob.mKnob = FindOrCreateKnob(key); + mTempKnob.mKey = key; + } + + bool PairedFindKnob(K key) { + mTempKnob.mKnob = FindKnob(key); + mTempKnob.mKey = key; + if (mTempKnob.mKnob) + return true; + else + return false; + } + + void PairedAddElem(E data) { + AddElem(mTempKnob.mKnob, data); + } + + // If 'data' doesn't exist, it ends quietly + void PairedRemoveElem(E data) { + RemoveElem(mTempKnob.mKnob, data); + } + + bool PairedFindElem(E data) { + return FindElem(mTempKnob.mKnob, data); + } + + // If 'data' doesn't exist, it ends quietly. + void PairedMoveElemToHead(E data) { + MoveElemToHead(mTempKnob.mKnob, data); + } + + E PairedFindFirstElem(bool &found) { + return FindFirstElem(mTempKnob.mKnob, found); + } + + // return num of elements in current temp knob. + // It's caller's duty to assure knob is not NULL. + unsigned PairedNumOfElem() { + return NumOfElem(mTempKnob.mKnob); + } + + // Return the idx-th element in knob. + // It's caller's duty to assure the validity of return value. + // It doesn't check validity here. + // Index starts from 0. + E PairedGetElemAtIndex(unsigned idx) { + return GetElemAtIndex(mTempKnob.mKnob, idx); + } + + // Reduce the element at index exc_idx. + // It's caller's duty to assure the element exists. + void PairedReduceElems(unsigned exc_idx) { + ReduceElems(mTempKnob.mKnob, exc_idx); + } + + void PairedSetKnobData(D d) { + mTempKnob.mKnob->mData = d; + } + + D PairedGetKnobData() { + return mTempKnob.mKnob->mData; + } + + K PairedGetKnobKey() { + return mTempKnob.mKey; + } + + ///////////////////////////////////////////////////////// + // Other functions + ///////////////////////////////////////////////////////// + + void Clear(){ + mTempKnob.mKnob = NULL; + mKnobs.clear(); + mMemPool.Clear(); + } + + void Release(){ + mMemPool.Release(); + } +}; + ////////////////////////////////////////////////////////////////////////////////////// // Tree // This is a regular tree. It simply maintains the basic operations of a tree, like diff --git a/src/MapleFE/shared/include/parser.h b/src/MapleFE/shared/include/parser.h index 26d4f73af706de7c750aa1326d749dc441219989..5b4a1072bc48fd4bce6da0d79d0ffe37756fbfdb 100644 --- a/src/MapleFE/shared/include/parser.h +++ b/src/MapleFE/shared/include/parser.h @@ -1,7 +1,8 @@ /* * Copyright (C) [2020-2022] Futurewei Technologies, Inc. All rights reverved. +* Copyright 2022 Tencent. All rights reverved. * -* OpenArkFE is licensed under the Mulan PSL v2. +* MapleFE is licensed under the Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. * You may obtain a copy of Mulan PSL v2 at: * diff --git a/src/MapleFE/shared/include/rule_summary.h b/src/MapleFE/shared/include/rule_summary.h index 014cc3ba470deedb3984ee98e199b4cf3bf6ff77..e6803bcd73b5f35eb5b870cb2e33d32d6c272942 100644 --- a/src/MapleFE/shared/include/rule_summary.h +++ b/src/MapleFE/shared/include/rule_summary.h @@ -1,3 +1,19 @@ +/* +* Copyright (C) [2020-2022] Futurewei Technologies, Inc. All rights reverved. +* Copyright 2022 Tencent. All rights reverved. +* +* MapleFE is licensed under the Mulan PSL v2. +* You can use this software according to the terms and conditions of the Mulan PSL v2. +* You may obtain a copy of Mulan PSL v2 at: +* +* http://license.coscl.org.cn/MulanPSL2 +* +* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR +* FIT FOR A PARTICULAR PURPOSE. +* See the Mulan PSL v2 for more details. +*/ + #ifndef __RULE_SUMMARY_H__ #define __RULE_SUMMARY_H__ #include "ruletable.h" diff --git a/src/MapleFE/shared/include/succ_match.h b/src/MapleFE/shared/include/succ_match.h index fe78a9ecb27f05f2b4480742b4f3c9998d4556f3..648dc0be294fdb3a04afba79679b5ecb3596c819 100644 --- a/src/MapleFE/shared/include/succ_match.h +++ b/src/MapleFE/shared/include/succ_match.h @@ -1,7 +1,8 @@ /* * Copyright (C) [2020] Futurewei Technologies, Inc. All rights reverved. +* Copyright 2022 Tencent. All rights reverved. * -* OpenArkFE is licensed under the Mulan PSL v2. +* MapleFE is licensed under the Mulan PSL v2. * You can use this software according to the terms and conditions of the Mulan PSL v2. * You may obtain a copy of Mulan PSL v2 at: * @@ -47,8 +48,8 @@ namespace maplefe { class AppealNode; class SuccMatch { private: - Guamian mNodes; - Guamian mMatches; + GuamianFast mNodes; + GuamianFast mMatches; public: SuccMatch(){} diff --git a/src/MapleFE/shared/src/ast_builder.cpp b/src/MapleFE/shared/src/ast_builder.cpp index 707f0e13e694266eb809fe5e0c69ad4f93ec9e8c..7cb6decc43606a03aee8fe5b64b8f1b523d8786a 100644 --- a/src/MapleFE/shared/src/ast_builder.cpp +++ b/src/MapleFE/shared/src/ast_builder.cpp @@ -3819,7 +3819,7 @@ TreeNode* ASTBuilder::BuildUnionUserType() { // assert, both children cannot be UnionUserType at the same time. MASSERT(!user_type); user_type = ut; - user_type->AddUnionInterType(child_a); + user_type->AddUnionInterType(child_a, true); } } @@ -3865,7 +3865,7 @@ TreeNode* ASTBuilder::BuildInterUserType() { // assert, both children cannot be UnionUserType at the same time. MASSERT(!user_type); user_type = ut; - user_type->AddUnionInterType(child_a); + user_type->AddUnionInterType(child_a, true); } } diff --git a/src/MapleFE/shared/src/ast_type.cpp b/src/MapleFE/shared/src/ast_type.cpp index 1ac736af9365f1263a8f29738d538c35fe56491d..74ab86984068083463faea968862b181c685d6b0 100644 --- a/src/MapleFE/shared/src/ast_type.cpp +++ b/src/MapleFE/shared/src/ast_type.cpp @@ -27,7 +27,7 @@ namespace maplefe { // UserTypeNode // ////////////////////////////////////////////////////////////////////////// -void UserTypeNode::AddUnionInterType(TreeNode *args) { +void UserTypeNode::AddUnionInterType(TreeNode *args, bool front) { if (args->IsIdentifier() || args->IsPrimType() || args->IsPrimArrayType() || @@ -45,6 +45,11 @@ void UserTypeNode::AddUnionInterType(TreeNode *args) { args->IsTemplateLiteral() || args->IsStruct()) { mUnionInterTypes.PushBack(args); + if (front) { + for(unsigned i = mUnionInterTypes.GetNum() - 1; i > 0; --i) + mUnionInterTypes.SetElem(i, mUnionInterTypes.ValueAtIndex(i-1)); + mUnionInterTypes.SetElem(0, args); + } SETPARENT(args); } else if (args->IsPass()) { PassNode *p = (PassNode*)args; diff --git a/src/MapleFE/shared/src/stringpool.cpp b/src/MapleFE/shared/src/stringpool.cpp index 046555147f14d2f83a2868c5c97209f68230d63b..3b6e292486e4199e33c3347bc6bc941298d08b41 100644 --- a/src/MapleFE/shared/src/stringpool.cpp +++ b/src/MapleFE/shared/src/stringpool.cpp @@ -187,31 +187,64 @@ const char *StringPool::GetStringFromStrIdx(unsigned idx) { // This is the public interface to setup AltStrIdxMap used for obfuscation // a name is mapped to a fixed length random unused name. -// two letters, [a-zA-Z] [a-zA-Z], which will cover over 2K names +// starting from 2-letter names, [a-zA-Z] [a-zA-Z], which will cover over 2K names // AA Aa AB Ab, ...., zz -#define LEN 2 -// -#define KIND 52 -#define SIZE KIND*KIND +// if not enough will extend to use 3-letter or 4-letter for over 7 million names void StringPool::SetAltStrIdxMap() { - unsigned size = mAltStrIdxSet.size(); + // starting from 2-letter names + unsigned len = 2; bool done = false; - char *A = (char*)malloc(LEN+1); - *(A+LEN) = 0; + + // names use [A-Z] and [a-z] total 52 letters + int k = 52; + + // total number of names can be handled for len = 4, 3, 2, 1 respectively + int Size[4] = {k*k*k*k, k*k*k, k*k, k}; + + // names, trailing '\0' + char A[5] = {0, 0, 0, 0, 0}; + + // names already encounted, either existing name or new names + std::unordered_set used; + for (auto stridx : mAltStrIdxSet) { done = false; while (!done) { - int r = rand() % (SIZE); - int t = r/KIND; - int s = r%KIND; + unsigned offset = 4 - len; + int mod = Size[offset]; + + int n = rand(); + int r = n % mod; + + // check if already encounted + if (used.find(r) != used.end()) { + // expand to use one more leter if close to limit + if (used.size() > mod - Size[offset + 1]) { + len++; + MASSERT(len < 5 && "Need more names"); + } + continue; + } + + // have un-encounted name + used.insert(r); - // first char, use upper case for odd number - bool odd = t%2; - *A = (odd ? 'A' : 'a') + t/2; + int q; + bool odd; + int i = 0; + while (i < len - 1) { + mod = Size[offset + 1 + i]; + q = r / mod; + r = r % mod; - // second char, use upper case for odd number - odd = s%2; - *(A+1) = (odd ? 'A' : 'a') + s/2; + // char, use upper case for odd number + odd = q%2; + A[i++] = (odd ? 'A' : 'a') + q/2; + } + + // last char, use upper case for odd number + odd = r%2; + A[i] = (odd ? 'A' : 'a') + r/2; unsigned size = GetSize(); unsigned alt = GetStrIdx(A); @@ -222,8 +255,6 @@ void StringPool::SetAltStrIdxMap() { } } } - - free(A); } void StringPool::Dump() { diff --git a/src/MapleFE/test/typescript/unit_tests/semicolon-missing16.ts.result b/src/MapleFE/test/typescript/unit_tests/semicolon-missing16.ts.result new file mode 100644 index 0000000000000000000000000000000000000000..5a85dd3971104c3fd5b7e648634c0a1e7b0ff316 --- /dev/null +++ b/src/MapleFE/test/typescript/unit_tests/semicolon-missing16.ts.result @@ -0,0 +1,15 @@ +Matched 46 tokens. +Matched 56 tokens. +============= Module =========== +== Sub Tree == +func func(arg) throws: + cond-branch cond:arg LT 1 + true branch : + return false branch : + + for ( ) + console.log(i) + return arg Mul 10 + +== Sub Tree == +console.log(func(3)) diff --git a/src/MapleFE/typescript/stmt.spec b/src/MapleFE/typescript/stmt.spec index a37f858835384b055fbb617bb90fcb8d420f9ca4..973ef07d7e202c5799a03a5f62d251d6bcf07ca3 100644 --- a/src/MapleFE/typescript/stmt.spec +++ b/src/MapleFE/typescript/stmt.spec @@ -1,6 +1,7 @@ # Copyright (C) [2021] Futurewei Technologies, Inc. All rights reverved. +# Copyright 2022 Tencent. All rights reverved. # -# OpenArkFE is licensed under the Mulan PSL v2. +# MapleFE is licensed under the Mulan PSL v2. # You can use this software according to the terms and conditions of the Mulan PSL v2. # You may obtain a copy of Mulan PSL v2 at: # @@ -1165,9 +1166,9 @@ rule BreakStatement : ONEOF( ## return ; ## return [no LineTerminator here] Expression[In, ?Yield] ; rule ReturnStatement :ONEOF("return" + ZEROORONE(';'), - "return" + Expression + ZEROORONE(';')) + "return" + NoLineTerminator + Expression + ZEROORONE(';')) attr.action.%1 : BuildReturn() - attr.action.%2 : BuildReturn(%2) + attr.action.%2 : BuildReturn(%3) ##----------------------------------- ##rule WithStatement[Yield, Return] :