From 801c9238cd0e025eba0957c9e7698461203ab786 Mon Sep 17 00:00:00 2001 From: gavin1012_hw Date: Wed, 26 Oct 2022 11:16:11 +0800 Subject: [PATCH] Speeding up hap-compiling via ts2panda Issue: I5XQTL Signed-off-by: gavin1012_hw Change-Id: I02e20ff97f75075435bd7865618d40bfb5773f11 --- merge_abc/src/mergeProgram.cpp | 2 +- ts2panda/src/cmdOptions.ts | 14 ++- ts2panda/src/compilerDriver.ts | 25 ++-- ts2panda/src/index.ts | 87 +++++++++++++- ts2panda/src/ts2panda.ts | 39 ++++++- ts2panda/ts2abc/main.cpp | 13 ++- ts2panda/ts2abc/ts2abc.cpp | 188 ++++++++++++++++++++----------- ts2panda/ts2abc/ts2abc.h | 4 +- ts2panda/ts2abc/ts2abc_options.h | 8 ++ 9 files changed, 288 insertions(+), 92 deletions(-) diff --git a/merge_abc/src/mergeProgram.cpp b/merge_abc/src/mergeProgram.cpp index 0254e24d19..3b4037d420 100644 --- a/merge_abc/src/mergeProgram.cpp +++ b/merge_abc/src/mergeProgram.cpp @@ -107,7 +107,7 @@ bool MergeProgram::AppendProtoFiles(const std::string &filePath, const std::stri } protoFiles.insert(protoFiles.end(), directoryFiles.begin(), directoryFiles.end()); } else { - std::cerr << "Input must be either a regular file or directory" << std::endl; + std::cerr << "The input path: " << fPath << " must be either a regular file or directory." << std::endl; return false; } diff --git a/ts2panda/src/cmdOptions.ts b/ts2panda/src/cmdOptions.ts index 99a1539d77..b63e338fae 100644 --- a/ts2panda/src/cmdOptions.ts +++ b/ts2panda/src/cmdOptions.ts @@ -21,7 +21,7 @@ import { LOGE } from "./log"; import * as path from "path"; import { execute } from "./base/util"; -const ts2pandaOptions = [ +export const ts2pandaOptions = [ { name: 'commonjs', alias: 'c', type: Boolean, defaultValue: false, description: "compile as commonJs module." }, { name: 'modules', alias: 'm', type: Boolean, defaultValue: false, description: "compile as module." }, { name: 'debug-log', alias: 'l', type: Boolean, defaultValue: false, description: "show info debug log and generate the json file." }, @@ -54,6 +54,7 @@ const ts2pandaOptions = [ { name: 'record-name', type: String, defaultValue: "", description: "specify the record name, this option can only be used when [merge-abc] is enabled." }, { name: 'output-proto', type: Boolean, defaultValue: false, description: "Output protoBin file. Default: false" }, { name: 'merge-abc', type: Boolean, defaultValue: false, description: "Compile as merge abc" }, + { name: 'input-file', type: String, defaultValue: "", description: "A file containing a list of source files to be compiled. Each line of this file should be constructed in such format: fileName;recordName;moduleType;sourceFile" }, ] @@ -145,6 +146,17 @@ export class CmdOptions { return true; } + static isCompileFilesList(): boolean { + if (!this.options) { + return false; + } + return this.options["input-file"].length != 0; + } + + static getCompileFilesList(): string { + return this.options["input-file"]; + } + static isCommonJs(): boolean { if (!this.options) { return false; diff --git a/ts2panda/src/compilerDriver.ts b/ts2panda/src/compilerDriver.ts index 8409b71ab9..dd06cadb15 100644 --- a/ts2panda/src/compilerDriver.ts +++ b/ts2panda/src/compilerDriver.ts @@ -92,8 +92,10 @@ export class CompilerDriver { } } - initiateTs2abcChildProcess() { - this.ts2abcProcess = initiateTs2abc([this.fileName]); + initiateTs2abcChildProcess(args: Array) { + this.ts2abcProcess = initiateTs2abc(args); + listenChildExit(this.getTs2abcProcess()); + listenErrorEvent(this.getTs2abcProcess()); } getTs2abcProcess(): any { @@ -163,7 +165,7 @@ export class CompilerDriver { checkDuplicateDeclaration(recorder); } - compile(node: ts.SourceFile): void { + compile(node: ts.SourceFile, ts2abcProcess?: any | undefined): void { CompilerDriver.isTsFile = CompilerDriver.isTypeScriptSourceFile(node); if (CmdOptions.showASTStatistics()) { let statics: number[] = new Array(ts.SyntaxKind.Count).fill(0); @@ -180,10 +182,12 @@ export class CompilerDriver { // initiate ts2abc if (!CmdOptions.isAssemblyMode()) { - this.initiateTs2abcChildProcess(); + if (ts2abcProcess) { + this.ts2abcProcess = ts2abcProcess; + } else { + this.initiateTs2abcChildProcess([this.fileName]); + } let ts2abcProc = this.getTs2abcProcess(); - listenChildExit(ts2abcProc); - listenErrorEvent(ts2abcProc); try { if (CmdOptions.isMergeAbc()) { @@ -200,9 +204,14 @@ export class CompilerDriver { Ts2Panda.dumpStringsArray(ts2abcProc); Ts2Panda.dumpConstantPool(ts2abcProc); Ts2Panda.dumpModuleRecords(ts2abcProc); - Ts2Panda.dumpTypeInfoRecord(ts2abcProc, true) + Ts2Panda.dumpTypeInfoRecord(ts2abcProc, true); + + if (ts2abcProcess) { + Ts2Panda.dumpOutputFileName(ts2abcProc, this.fileName); + } else { + terminateWritePipe(ts2abcProc); + } - terminateWritePipe(ts2abcProc); if (CmdOptions.isEnableDebugLog()) { let jsonFileName = this.fileName.substring(0, this.fileName.lastIndexOf(".")).concat(".json"); writeFileSync(jsonFileName, Ts2Panda.jsonString); diff --git a/ts2panda/src/index.ts b/ts2panda/src/index.ts index 7fb5dca084..41c8d54f0a 100644 --- a/ts2panda/src/index.ts +++ b/ts2panda/src/index.ts @@ -16,7 +16,8 @@ import * as path from "path"; import * as ts from "typescript"; import * as fs from "fs"; -import { CmdOptions } from "./cmdOptions"; +import { CmdOptions, ts2pandaOptions } from "./cmdOptions"; +import commandLineArgs from "command-line-args"; import { CompilerDriver } from "./compilerDriver"; import * as diag from "./diagnostic"; import * as jshelpers from "./jshelpers"; @@ -32,7 +33,8 @@ import { isBase64Str, transformCommonjsModule, getRecordName, - getOutputBinName + getOutputBinName, + terminateWritePipe } from "./base/util"; function checkIsGlobalDeclaration(sourceFile: ts.SourceFile) { @@ -61,7 +63,7 @@ function generateDTs(node: ts.SourceFile, options: ts.CompilerOptions) { compilerDriver.showStatistics(); } -function main(fileNames: string[], options: ts.CompilerOptions) { +function main(fileNames: string[], options: ts.CompilerOptions, cmdArgsSet?: Map) { const host = ts.createCompilerHost(options); if (!CmdOptions.needGenerateTmpFile()) { host.writeFile = () => {}; @@ -89,6 +91,12 @@ function main(fileNames: string[], options: ts.CompilerOptions) { return; } + let initCompilerDriver; + if (CmdOptions.isCompileFilesList()) { + initCompilerDriver = new CompilerDriver("", ""); + initCompilerDriver.initiateTs2abcChildProcess(["--multi-programs-pipe"]); + } + let emitResult = program.emit( undefined, undefined, @@ -99,6 +107,17 @@ function main(fileNames: string[], options: ts.CompilerOptions) { // @ts-ignore (ctx: ts.TransformationContext) => { return (node: ts.SourceFile) => { + if (CmdOptions.isCompileFilesList()) { + let specifiedCmdArgs = cmdArgsSet.get(node.fileName); + if (specifiedCmdArgs === undefined) { + return node; + } + let originProcessArgs = process.argv.slice(0); + Array.prototype.splice.apply(process.argv, specifiedCmdArgs); + //@ts-ignore + CmdOptions.options = commandLineArgs(ts2pandaOptions, { partial: true }); + process.argv = originProcessArgs.slice(0); + } let outputBinName = getOutputBinName(node); let compilerDriver = new CompilerDriver(outputBinName, getRecordName(node)); compilerDriver.compileForSyntaxCheck(node); @@ -110,8 +129,13 @@ function main(fileNames: string[], options: ts.CompilerOptions) { // @ts-ignore (ctx: ts.TransformationContext) => { return (node: ts.SourceFile) => { + if (CmdOptions.isCompileFilesList()) { + let specifiedCmdArgs = cmdArgsSet.get(node.fileName); + if (specifiedCmdArgs == undefined) { + return node; + } + } makeAutoGeneratedNodeName(node); - if (ts.getEmitHelpers(node)) { let newStatements = []; ts.getEmitHelpers(node)?.forEach( @@ -133,7 +157,11 @@ function main(fileNames: string[], options: ts.CompilerOptions) { let compilerDriver = new CompilerDriver(outputBinName, getRecordName(node)); CompilerDriver.srcNode = node; setGlobalStrict(jshelpers.isEffectiveStrictModeSourceFile(node, options)); - compilerDriver.compile(node); + if (CmdOptions.isCompileFilesList()) { + compilerDriver.compile(node, initCompilerDriver.getTs2abcProcess()); + } else { + compilerDriver.compile(node); + } compilerDriver.showStatistics(); return node; } @@ -142,6 +170,10 @@ function main(fileNames: string[], options: ts.CompilerOptions) { } ); + if (CmdOptions.isCompileFilesList()) { + terminateWritePipe(initCompilerDriver.getTs2abcProcess()); + } + let allDiagnostics = ts .getPreEmitDiagnostics(program) .concat(emitResult.diagnostics); @@ -155,6 +187,46 @@ function main(fileNames: string[], options: ts.CompilerOptions) { }); } +function transformSourcefilesList(parsed: ts.ParsedCommandLine | undefined) { + let inputFile = CmdOptions.getCompileFilesList(); + let sourceFileInfoArray = []; + try { + sourceFileInfoArray = fs.readFileSync(inputFile).toString().split("\n"); + } catch(err) { + throw err; + } + if (sourceFileInfoArray.length == 0) return; + + let files: string[] = parsed.fileNames; + let cmdArgsSet = new Map(); + + for (let i = 0; i < sourceFileInfoArray.length; i++) { + let sourceFileInfo = sourceFileInfoArray[i].split(";"); + if (sourceFileInfo.length != 4) { + throw new Error("Input info for each file need \"fileName;recordName;moduleType;sourceFile\" style."); + } + + let inputFileName = sourceFileInfo[0]; + files.unshift(inputFileName); + let specifiedCmdArgs = ["--record-name", sourceFileInfo[1]]; + switch (sourceFileInfo[2]) { + case "esm": + specifiedCmdArgs.push.apply(specifiedCmdArgs, ["-m"]); + break; + case "commonjs": + specifiedCmdArgs.push.apply(specifiedCmdArgs, ["-c"]); + break; + default: + throw new Error("The third info should be \"esm\" or \"commonjs\"."); + } + specifiedCmdArgs.push.apply(specifiedCmdArgs, ["--source-file", sourceFileInfo[3]]); + specifiedCmdArgs.unshift(2, 0); + cmdArgsSet.set(inputFileName, specifiedCmdArgs); + } + + main(files, parsed.options, cmdArgsSet); +} + let generatedVarId = 0; function makeAutoGeneratedNodeName(node: ts.Node) { @@ -339,6 +411,7 @@ function compileWatchExpression(jsFileName: string, errorMsgFileName: string, op // @ts-ignore (ctx: ts.TransformationContext) => { return (node: ts.SourceFile) => { + makeAutoGeneratedNodeName(node); if (ts.getEmitHelpers(node)) { let newStatements = []; ts.getEmitHelpers(node)?.forEach( @@ -470,6 +543,10 @@ function run(args: string[], options?: ts.CompilerOptions): void { updateWatchJsFile(); return; } + if (CmdOptions.isCompileFilesList()) { + transformSourcefilesList(parsed); + return; + } main(parsed.fileNames.concat(CmdOptions.getIncludedFiles()), parsed.options); } catch (err) { diff --git a/ts2panda/src/ts2panda.ts b/ts2panda/src/ts2panda.ts index 629511fd0e..430e5dc512 100644 --- a/ts2panda/src/ts2panda.ts +++ b/ts2panda/src/ts2panda.ts @@ -58,6 +58,7 @@ import { isFunctionLikeDeclaration } from "./syntaxCheckHelper"; import { getLiteralKey } from "./base/util"; const dollarSign: RegExp = /\$/g; +const starSign: RegExp = /\*/g; const JsonType = { "function": 0, @@ -67,7 +68,8 @@ const JsonType = { "module": 4, "options": 5, 'type_info': 6, - 'record_name': 7 + 'record_name': 7, + 'output_filename': 8 }; export class Ts2Panda { static strings: Set = new Set(); @@ -169,6 +171,7 @@ export class Ts2Panda { let jsonStrUnicode = escapeUnicode(JSON.stringify(strObject, null, 2)); jsonStrUnicode = "$" + jsonStrUnicode.replace(dollarSign, '#$') + "$"; + jsonStrUnicode = jsonStrUnicode.replace(starSign, '#*'); if (CmdOptions.isEnableDebugLog()) { Ts2Panda.jsonString += jsonStrUnicode; } @@ -205,6 +208,7 @@ export class Ts2Panda { } let jsonLiteralArrUnicode = escapeUnicode(JSON.stringify(literalArrayObject, null, 2)); jsonLiteralArrUnicode = "$" + jsonLiteralArrUnicode.replace(dollarSign, '#$') + "$"; + jsonLiteralArrUnicode = jsonLiteralArrUnicode.replace(starSign, '#*'); if (CmdOptions.isEnableDebugLog()) { Ts2Panda.jsonString += jsonLiteralArrUnicode; } @@ -226,10 +230,12 @@ export class Ts2Panda { "display_typeinfo": CmdOptions.getDisplayTypeinfo(), "is_dts_file": isGlobalDeclare(), "output-proto": CmdOptions.isOutputproto(), - "record_type": enableRecordType + "record_type": enableRecordType, + "input-file": CmdOptions.getCompileFilesList() }; let jsonOpt = JSON.stringify(options, null, 2); jsonOpt = "$" + jsonOpt.replace(dollarSign, '#$') + "$"; + jsonOpt = jsonOpt.replace(starSign, '#*'); if (CmdOptions.isEnableDebugLog()) { Ts2Panda.jsonString += jsonOpt; } @@ -243,6 +249,7 @@ export class Ts2Panda { } let jsonRecord = escapeUnicode(JSON.stringify(record, null, 2)); jsonRecord = "$" + jsonRecord.replace(dollarSign, '#$') + "$"; + jsonRecord = jsonRecord.replace(starSign, '#*'); if (CmdOptions.isEnableDebugLog()) { Ts2Panda.jsonString += jsonRecord; } @@ -439,6 +446,7 @@ export class Ts2Panda { } let jsonFuncUnicode = escapeUnicode(JSON.stringify(funcObject, null, 2)); jsonFuncUnicode = "$" + jsonFuncUnicode.replace(dollarSign, '#$') + "$"; + jsonFuncUnicode = jsonFuncUnicode.replace(starSign, '#*'); if (CmdOptions.isEnableDebugLog()) { Ts2Panda.jsonString += jsonFuncUnicode; } @@ -453,6 +461,7 @@ export class Ts2Panda { }; let jsonModuleUnicode = escapeUnicode(JSON.stringify(moduleObject, null, 2)); jsonModuleUnicode = "$" + jsonModuleUnicode.replace(dollarSign, '#$') + "$"; + jsonModuleUnicode = jsonModuleUnicode.replace(starSign, '#*'); if (CmdOptions.isEnableDebugLog()) { Ts2Panda.jsonString += jsonModuleUnicode; } @@ -474,12 +483,30 @@ export class Ts2Panda { 't': JsonType.type_info, 'ti': typeInfo }; - let jsonModuleUnicode = escapeUnicode(JSON.stringify(typeInfoObject, null, 2)); - jsonModuleUnicode = "$" + jsonModuleUnicode.replace(dollarSign, '#$') + "$"; + let jsonTypeInfoUnicode = escapeUnicode(JSON.stringify(typeInfoObject, null, 2)); + jsonTypeInfoUnicode = "$" + jsonTypeInfoUnicode.replace(dollarSign, '#$') + "$"; + jsonTypeInfoUnicode = jsonTypeInfoUnicode.replace(starSign, '#*'); if (CmdOptions.isEnableDebugLog()) { - Ts2Panda.jsonString += jsonModuleUnicode; + Ts2Panda.jsonString += jsonTypeInfoUnicode; } - ts2abc.stdio[3].write(jsonModuleUnicode + '\n'); + ts2abc.stdio[3].write(jsonTypeInfoUnicode + '\n'); + } + + static dumpOutputFileName(ts2abc: any, outputFileName: string): void { + let outputFileNameObject = { + "t": JsonType.output_filename, + "ofn": outputFileName + } + + let jsonOutputFileNameUnicode = escapeUnicode(JSON.stringify(outputFileNameObject, null, 2)); + jsonOutputFileNameUnicode = "$" + jsonOutputFileNameUnicode.replace(dollarSign, '#$') + "$"; + jsonOutputFileNameUnicode = jsonOutputFileNameUnicode.replace(starSign, '#*'); + if (CmdOptions.isEnableDebugLog()) { + Ts2Panda.jsonString += jsonOutputFileNameUnicode; + } + ts2abc.stdio[3].write(jsonOutputFileNameUnicode + '\n'); + // seperator between program + ts2abc.stdio[3].write("*" + '\n'); } static clearDumpData() { diff --git a/ts2panda/ts2abc/main.cpp b/ts2panda/ts2abc/main.cpp index 7e315e564c..d5d2230339 100644 --- a/ts2panda/ts2abc/main.cpp +++ b/ts2panda/ts2abc/main.cpp @@ -86,7 +86,7 @@ int main(int argc, const char *argv[]) return panda::ts2abc::RETURN_FAILED; } - std::string usage = "Usage: ts2abc [OPTIONS]... [ARGS]..."; + std::string usage = "Usage: js2abc [OPTIONS]... [ARGS]..."; if (options.GetHelpArg()) { std::cout << usage << std::endl; std::cout << argParser.GetHelpString(); @@ -116,9 +116,18 @@ int main(int argc, const char *argv[]) return panda::ts2abc::RETURN_FAILED; } + std::string optLogLevel(options.GetOptLogLevelArg()); + + if (options.IsMultiProgramsPipe()) { + if (!panda::ts2abc::GenerateProgramsFromPipe(options)) { + std::cerr << "call GenerateProgramsFromPipe fail" << std::endl; + return panda::ts2abc::RETURN_FAILED; + } + return panda::ts2abc::RETURN_SUCCESS; + } + std::string output; std::string data = ""; - std::string optLogLevel(options.GetOptLogLevelArg()); if (Preprocess(options, argParser, output, data, usage) == panda::ts2abc::RETURN_FAILED) { return panda::ts2abc::RETURN_FAILED; diff --git a/ts2panda/ts2abc/ts2abc.cpp b/ts2panda/ts2abc/ts2abc.cpp index 2e564efc01..9b0be99403 100644 --- a/ts2panda/ts2abc/ts2abc.cpp +++ b/ts2panda/ts2abc/ts2abc.cpp @@ -51,6 +51,8 @@ static constexpr const char* TSTYPE_ANNO_RECORD_NAME = "_ESTypeAnnotation"; static constexpr const char* TSTYPE_ANNO_ELEMENT_NAME = "_TypeOfInstruction"; std::string g_compilerOutputProto = ""; std::string g_recordName = ""; +std::string g_outputFileName = ""; +bool g_isStartDollar = true; constexpr uint32_t LITERALBUFFERINDEXOFFSET = 100; uint32_t MAX_UINT8 = static_cast(std::numeric_limits::max()); bool g_isOutputProto = false; @@ -1329,6 +1331,12 @@ static int ParseSmallPieceJson(const std::string &subJson, panda::pandasm::Progr } break; } + case static_cast(JsonType::OUTPUTFILENAME): { + if (rootValue.isMember("ofn") && rootValue["ofn"].isString()) { + g_outputFileName = rootValue["ofn"].asString(); + } + break; + } default: { std::cerr << "Unreachable json type: " << type << std::endl; return RETURN_FAILED; @@ -1345,23 +1353,23 @@ static bool ParseData(const std::string &data, panda::pandasm::Program &prog) } size_t pos = 0; - bool isStartDollar = true; for (size_t idx = 0; idx < data.size(); idx++) { - if (data[idx] == '$' && (idx ==0 || data[idx - 1] != '#')) { - if (isStartDollar) { + if (data[idx] == '$' && (idx == 0 || data[idx - 1] != '#')) { + if (g_isStartDollar) { pos = idx + 1; - isStartDollar = false; + g_isStartDollar = false; continue; } std::string subJson = data.substr(pos, idx - pos); ReplaceAllDistinct(subJson, "#$", "$"); + ReplaceAllDistinct(subJson, "#*", "*"); if (ParseSmallPieceJson(subJson, prog)) { std::cerr << "fail to parse stringify json" << std::endl; return false; } - isStartDollar = true; + g_isStartDollar = true; } } @@ -1385,31 +1393,115 @@ static bool IsStartOrEndPosition(int idx, char *buff, const std::string &data) return false; } -static bool HandleBuffer(const int &ret, bool &isStartDollar, char *buff, std::string &data, panda::pandasm::Program &prog) +static bool EmitProgram(const std::string &output, int optLevel, std::string optLogLevel, panda::pandasm::Program &prog) +{ + if (g_isOutputProto) { + g_compilerOutputProto = output.substr(0, output.find_last_of(".") + 1).append(PROTO_BIN_SUFFIX); + } + +#ifdef ENABLE_BYTECODE_OPT + if (g_optLevel != static_cast(OptLevel::O_LEVEL0) || optLevel != static_cast(OptLevel::O_LEVEL0)) { + optLogLevel = (optLogLevel != "error") ? optLogLevel : g_optLogLevel; + + if (g_optLogLevel != "error") { + panda::Logger::ComponentMask mask; + mask.set(panda::Logger::Component::ASSEMBLER); + mask.set(panda::Logger::Component::BYTECODE_OPTIMIZER); + mask.set(panda::Logger::Component::COMPILER); + panda::Logger::InitializeStdLogging(panda::Logger::LevelFromString(optLogLevel), mask); + } + + bool emitDebugInfo = true; + std::map stat; + std::map *statp = nullptr; + panda::pandasm::AsmEmitter::PandaFileToPandaAsmMaps maps {}; + panda::pandasm::AsmEmitter::PandaFileToPandaAsmMaps* mapsp = &maps; + + if (!panda::pandasm::AsmEmitter::Emit(output.c_str(), prog, statp, mapsp, emitDebugInfo)) { + std::cerr << "Failed to emit binary data: " << panda::pandasm::AsmEmitter::GetLastError() << std::endl; + return false; + } + + panda::bytecodeopt::OptimizeBytecode(&prog, mapsp, output.c_str(), true); + + if (g_compilerOutputProto.size() > 0) { + panda::proto::ProtobufSnapshotGenerator::GenerateSnapshot(prog, g_compilerOutputProto); + return true; + } + + if (!panda::pandasm::AsmEmitter::Emit(output.c_str(), prog, statp, mapsp, emitDebugInfo)) { + std::cerr << "Failed to emit binary data: " << panda::pandasm::AsmEmitter::GetLastError() << std::endl; + return false; + } + return true; + } +#endif + if (g_compilerOutputProto.size() > 0) { + panda::proto::ProtobufSnapshotGenerator::GenerateSnapshot(prog, g_compilerOutputProto); + return true; + } + + if (!panda::pandasm::AsmEmitter::Emit(output.c_str(), prog, nullptr)) { + std::cerr << "Failed to emit binary data: " << panda::pandasm::AsmEmitter::GetLastError() << std::endl; + return false; + } + + Logd("Successfully generated: %s\n", output.c_str()); + return true; +} + +static bool EmitAndRestoreProgram(panda::pandasm::Program &prog, panda::ts2abc::Options options) +{ + if (!EmitProgram(g_outputFileName, options.GetOptLevelArg(), options.GetOptLogLevelArg(), prog)) { + std::cerr << "fail to emit porgram " << g_outputFileName << " in HandleBuffer" << std::endl; + return false; + } + prog = panda::pandasm::Program(); + prog.lang = panda::pandasm::extensions::Language::ECMASCRIPT; + return true; +} + +static bool HandleBuffer(const int &ret, char *buff, std::string &data, panda::pandasm::Program &prog, + panda::ts2abc::Options options) { uint32_t startPos = 0; + if (options.IsMultiProgramsPipe() && ((buff[0] == '*' && data.back() != '#') || + (buff[0] == '\n' && buff[1] == '*'))) { + if (!EmitAndRestoreProgram(prog, options)) { + std::cerr << "fail to emit and restore program" << std::endl; + return false; + } + } + for (int idx = 0; idx < ret; idx++) { if (IsStartOrEndPosition(idx, buff, data)) { - if (isStartDollar) { + if (g_isStartDollar) { startPos = idx + 1; - isStartDollar = false; + g_isStartDollar = false; continue; } std::string substr(buff + startPos, buff + idx); data += substr; ReplaceAllDistinct(data, "#$", "$"); + ReplaceAllDistinct(data, "#*", "*"); if (ParseSmallPieceJson(data, prog)) { std::cerr << "fail to parse stringify json" << std::endl; return false; } - isStartDollar = true; + g_isStartDollar = true; // clear data after parsing data.clear(); + // idx: $, then idx + 1:'\n', so comparing buff[idx + 2] with '*' + if (options.IsMultiProgramsPipe() && idx < ret - 2 && buff[idx + 2] == '*' && + !EmitAndRestoreProgram(prog, options)) { + std::cerr << "fail to emit and restore program" << std::endl; + return false; + } } } - if (!isStartDollar) { + if (!g_isStartDollar) { std::string substr(buff + startPos, buff + ret); data += substr; } @@ -1417,10 +1509,9 @@ static bool HandleBuffer(const int &ret, bool &isStartDollar, char *buff, std::s return true; } -static bool ReadFromPipe(panda::pandasm::Program &prog) +static bool ReadFromPipe(panda::pandasm::Program &prog, panda::ts2abc::Options options) { std::string data; - bool isStartDollar = true; const size_t bufSize = 4096; // the parent process open a pipe to this child process with fd of 3 const size_t fd = 3; @@ -1435,7 +1526,7 @@ static bool ReadFromPipe(panda::pandasm::Program &prog) } buff[ret] = '\0'; - if (!HandleBuffer(ret, isStartDollar, buff, data, prog)) { + if (!HandleBuffer(ret, buff, data, prog, options)) { std::cerr << "fail to handle buffer" << std::endl; return false; } @@ -1445,6 +1536,19 @@ static bool ReadFromPipe(panda::pandasm::Program &prog) return true; } +bool GenerateProgramsFromPipe(panda::ts2abc::Options options) +{ + panda::pandasm::Program prog = panda::pandasm::Program(); + prog.lang = panda::pandasm::extensions::Language::ECMASCRIPT; + + if (!ReadFromPipe(prog, options)) { + std::cerr << "fail to parse programs from Pipe!" << std::endl; + return false; + } + + return true; +} + bool GenerateProgram([[maybe_unused]] const std::string &data, const std::string &output, panda::ts2abc::Options options) { @@ -1455,7 +1559,7 @@ bool GenerateProgram([[maybe_unused]] const std::string &data, const std::string prog.lang = panda::pandasm::extensions::Language::ECMASCRIPT; if (isParsingFromPipe) { - if (!ReadFromPipe(prog)) { + if (!ReadFromPipe(prog, options)) { std::cerr << "fail to parse Pipe!" << std::endl; return false; } @@ -1468,59 +1572,7 @@ bool GenerateProgram([[maybe_unused]] const std::string &data, const std::string Logd("parsing done, calling pandasm\n"); - if (g_isOutputProto) { - g_compilerOutputProto = output.substr(0, output.find_last_of(".") + 1).append(PROTO_BIN_SUFFIX); - } - -#ifdef ENABLE_BYTECODE_OPT - if (g_optLevel != static_cast(OptLevel::O_LEVEL0) || optLevel != static_cast(OptLevel::O_LEVEL0)) { - optLogLevel = (optLogLevel != "error") ? optLogLevel : g_optLogLevel; - - if (g_optLogLevel != "error") { - panda::Logger::ComponentMask mask; - mask.set(panda::Logger::Component::ASSEMBLER); - mask.set(panda::Logger::Component::BYTECODE_OPTIMIZER); - mask.set(panda::Logger::Component::COMPILER); - panda::Logger::InitializeStdLogging(panda::Logger::LevelFromString(optLogLevel), mask); - } - - bool emitDebugInfo = true; - std::map stat; - std::map *statp = nullptr; - panda::pandasm::AsmEmitter::PandaFileToPandaAsmMaps maps {}; - panda::pandasm::AsmEmitter::PandaFileToPandaAsmMaps* mapsp = &maps; - - if (!panda::pandasm::AsmEmitter::Emit(output.c_str(), prog, statp, mapsp, emitDebugInfo)) { - std::cerr << "Failed to emit binary data: " << panda::pandasm::AsmEmitter::GetLastError() << std::endl; - return false; - } - - panda::bytecodeopt::OptimizeBytecode(&prog, mapsp, output.c_str(), true); - - if (g_compilerOutputProto.size() > 0) { - panda::proto::ProtobufSnapshotGenerator::GenerateSnapshot(prog, g_compilerOutputProto); - return true; - } - - if (!panda::pandasm::AsmEmitter::Emit(output.c_str(), prog, statp, mapsp, emitDebugInfo)) { - std::cerr << "Failed to emit binary data: " << panda::pandasm::AsmEmitter::GetLastError() << std::endl; - return false; - } - return true; - } -#endif - if (g_compilerOutputProto.size() > 0) { - panda::proto::ProtobufSnapshotGenerator::GenerateSnapshot(prog, g_compilerOutputProto); - return true; - } - - if (!panda::pandasm::AsmEmitter::Emit(output.c_str(), prog, nullptr)) { - std::cerr << "Failed to emit binary data: " << panda::pandasm::AsmEmitter::GetLastError() << std::endl; - return false; - } - - Logd("Successfully generated: %s\n", output.c_str()); - return true; + return EmitProgram(output, optLevel, optLogLevel, prog); } bool CompileNpmEntries(const std::string &input, const std::string &output) @@ -1532,7 +1584,7 @@ bool CompileNpmEntries(const std::string &input, const std::string &output) } auto fpath = inputAbs.Value(); if (panda::os::file::File::IsRegularFile(fpath) == false) { - std::cerr << "Input must be either a regular file or a directory" << std::endl; + std::cerr << "Input: " << fpath << " must be either a regular file or a directory" << std::endl; return false; } diff --git a/ts2panda/ts2abc/ts2abc.h b/ts2panda/ts2abc/ts2abc.h index 9288b91c27..dfcee1e337 100644 --- a/ts2panda/ts2abc/ts2abc.h +++ b/ts2panda/ts2abc/ts2abc.h @@ -37,7 +37,8 @@ enum class JsonType { MODULE, OPTIONS, TYPEINFO, - RECORDNAME + RECORDNAME, + OUTPUTFILENAME }; constexpr int RETURN_SUCCESS = 0; @@ -51,6 +52,7 @@ enum class OptLevel { bool HandleJsonFile(const std::string &input, std::string &data); bool GenerateProgram(const std::string &data, const std::string &output, panda::ts2abc::Options options); +bool GenerateProgramsFromPipe(panda::ts2abc::Options options); bool CompileNpmEntries(const std::string &input, const std::string &output); bool GetDebugLog(); void ParseLogEnable(const Json::Value &rootValue); diff --git a/ts2panda/ts2abc/ts2abc_options.h b/ts2panda/ts2abc/ts2abc_options.h index 41dafe7c45..e08e66ac0a 100755 --- a/ts2panda/ts2abc/ts2abc_options.h +++ b/ts2panda/ts2abc/ts2abc_options.h @@ -41,6 +41,7 @@ namespace panda::ts2abc { parser->Add(&compile_by_pipe_arg_); parser->Add(&compile_npm_entries_); parser->Add(&compiler_output_proto_); + parser->Add(&multi_programs_pipe_); parser->EnableTail(); parser->PushBackTail(&Tail_Arg1_arg_); parser->PushBackTail(&Tail_Arg2_arg_); @@ -181,6 +182,11 @@ namespace panda::ts2abc { return compiler_output_proto_.WasSet(); } + bool IsMultiProgramsPipe() const + { + return multi_programs_pipe_.GetValue(); + } + std::string GetTailArg1() const { return Tail_Arg1_arg_.GetValue(); @@ -243,6 +249,8 @@ namespace panda::ts2abc { R"(Compile npm entries info into an abc file)"}; panda::PandArg compiler_output_proto_{ "output-proto", false, R"(Output protoBin file)"}; + panda::PandArg multi_programs_pipe_{ "multi-programs-pipe", false, + R"(Genrate programs by single pipe)"}; panda::PandArg Tail_Arg1_arg_{ "ARG_1", "", R"(Path to input(json file) or path to output(ark bytecode)" " when 'compile-by-pipe' enabled)"}; -- Gitee