From 124e0404e91dc27a04d65cce1f70059d19ed162d Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Fri, 23 Sep 2022 10:52:45 +0800 Subject: [PATCH 1/3] pick gen abc plugin change to js Signed-off-by: zhangrengao Change-Id: I44cc39b5e91954f2b63f4202736f8c80d17de45c --- compiler/src/gen_abc.ts | 8 +-- compiler/src/gen_abc_plugin.ts | 95 +++++++++++++++++++--------------- 2 files changed, 57 insertions(+), 46 deletions(-) diff --git a/compiler/src/gen_abc.ts b/compiler/src/gen_abc.ts index 6dbb3cf..83f79f8 100644 --- a/compiler/src/gen_abc.ts +++ b/compiler/src/gen_abc.ts @@ -31,9 +31,9 @@ const reset: string = '\u001b[39m'; function js2abcByWorkers(jsonInput: string, cmd: string): Promise { const inputPaths: any = JSON.parse(jsonInput); for (let i = 0; i < inputPaths.length; ++i) { - const input: string = inputPaths[i].path; + const input: string = inputPaths[i].path.replace(/\.temp\.js$/, "_.js"); const cacheOutputPath: string = inputPaths[i].cacheOutputPath; - const cacheAbcFilePath: string = cacheOutputPath.replace(/\_.js$/, ".abc"); + const cacheAbcFilePath: string = cacheOutputPath.replace(/\.temp\.js$/, ".abc"); const singleCmd: any = `${cmd} "${cacheOutputPath}" -o "${cacheAbcFilePath}" --source-file "${input}"`; logger.debug('gen abc cmd is: ', singleCmd, ' ,file size is:', inputPaths[i].size, ' byte'); try { @@ -50,9 +50,9 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { function es2abcByWorkers(jsonInput: string, cmd: string): Promise { const inputPaths: any = JSON.parse(jsonInput); for (let i = 0; i < inputPaths.length; ++i) { - const input: string = inputPaths[i].path; + const input: string = inputPaths[i].path.replace(/\.temp\.js$/, "_.js"); const cacheOutputPath: string = inputPaths[i].cacheOutputPath; - const cacheAbcFilePath: string = cacheOutputPath.replace(/\_.js$/, ".abc"); + const cacheAbcFilePath: string = cacheOutputPath.replace(/\.temp\.js$/, ".abc"); const singleCmd: any = `${cmd} "${cacheOutputPath}" --output "${cacheAbcFilePath}" --source-file "${input}"`; logger.debug('gen abc cmd is: ', singleCmd, ' ,file size is:', inputPaths[i].size, ' byte'); try { diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index cf67942..196d489 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -49,10 +49,10 @@ import { EXTNAME_JS_MAP, TS2ABC, ES2ABC, - TEMPORARY + TEMPORARY, + SUCCESS } from './pre_define'; -const firstFileEXT: string = '_.js'; const genAbcScript: string = 'gen_abc.js'; const genModuleAbcScript: string = 'gen_module_abc.js'; let output: string; @@ -176,8 +176,8 @@ export class GenAbcPlugin { // choose *.js if (output && path.extname(key) === EXTNAME_JS) { const newContent: string = compilation.assets[key].source(); - const keyPath: string = key.replace(/\.js$/, firstFileEXT); - writeFileSync(newContent, path.resolve(output, keyPath), key); + const keyPath: string = key.replace(/\.js$/, ".temp.js"); + writeFileSync(newContent, output, keyPath, key); } }); }); @@ -356,16 +356,15 @@ function processEntryToGenAbc(entryInfos: Map): void { } } -function writeFileSync(inputString: string, output: string, jsBundleFile: string): void { +function writeFileSync(inputString: string, buildPath: string, keyPath: string, jsBundleFile: string): void { + let output = path.resolve(buildPath, keyPath); let parent: string = path.join(output, '..'); if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) { mkDir(parent); } - let buildParentPath: string = path.join(projectConfig.buildPath, '..'); - let sufStr: string = output.replace(buildParentPath, ''); let cacheOutputPath: string = ""; if (process.env.cachePath) { - cacheOutputPath = path.join(process.env.cachePath, TEMPORARY, sufStr); + cacheOutputPath = path.join(process.env.cachePath, TEMPORARY, keyPath); } else { cacheOutputPath = output; } @@ -474,6 +473,9 @@ function initAbcEnv() : string[] { } function invokeCluterModuleToAbc(): void { + if (projectConfig.isPreview) { + process.exitCode = SUCCESS; + } filterIntermediateModuleByHashJson(buildPathInfo, moduleInfos); filterModuleInfos.forEach(moduleInfo => { if (moduleInfo.isCommonJs) { @@ -599,6 +601,9 @@ function judgeModuleWorkersToGenAbc(callback): void { } function invokeWorkersToGenAbc(): void { + if (projectConfig.isPreview) { + process.exitCode = SUCCESS; + } let cmdPrefix: string = ''; let maxWorkerNumber: number = 3; @@ -647,9 +652,9 @@ function invokeWorkersToGenAbc(): void { } count_++; if (count_ === workerNumber) { - writeHashJson(); - clearGlobalInfo(); + // for preview of with incre compile if (projectConfig.isPreview) { + processExtraAssetForBundle(); console.info(red, 'COMPILE RESULT:SUCCESS ', reset); } } @@ -657,13 +662,14 @@ function invokeWorkersToGenAbc(): void { }); process.on('exit', (code) => { - intermediateJsBundle.forEach((item) => { - const input = item.path; - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - }); + // for build options + processExtraAssetForBundle(); }); + + // for preview of without incre compile + if (workerNumber === 0 && projectConfig.isPreview) { + processExtraAssetForBundle(); + } } } @@ -786,10 +792,8 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil jsonObject = JSON.parse(jsonFile); fileterIntermediateJsBundle = []; for (let i = 0; i < inputPaths.length; ++i) { - const input: string = inputPaths[i].path; - const abcPath: string = input.replace(/_.js$/, EXTNAME_ABC); const cacheOutputPath: string = inputPaths[i].cacheOutputPath; - const cacheAbcFilePath: string = cacheOutputPath.replace(/\_.js$/, '.abc'); + const cacheAbcFilePath: string = cacheOutputPath.replace(/\.temp\.js$/, '.abc'); if (!fs.existsSync(cacheOutputPath)) { logger.error(red, `ETS:ERROR ${cacheOutputPath} is lost`, reset); process.exitCode = FAIL; @@ -801,10 +805,6 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil if (jsonObject[cacheOutputPath] === hashInputContentData && jsonObject[cacheAbcFilePath] === hashAbcContentData) { updateJsonObject[cacheOutputPath] = hashInputContentData; updateJsonObject[cacheAbcFilePath] = hashAbcContentData; - if (!fs.existsSync(abcPath)) { - mkdirsSync(path.dirname(abcPath)); - fs.copyFileSync(cacheAbcFilePath, abcPath); - } } else { fileterIntermediateJsBundle.push(inputPaths[i]); } @@ -819,10 +819,8 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil function writeHashJson(): void { for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { - const input:string = fileterIntermediateJsBundle[i].path; - const abcPath: string = input.replace(/_.js$/, EXTNAME_ABC); const cacheOutputPath: string = fileterIntermediateJsBundle[i].cacheOutputPath; - const cacheAbcFilePath: string = cacheOutputPath.replace(/\_.js$/, '.abc'); + const cacheAbcFilePath: string = cacheOutputPath.replace(/\.temp\.js$/, '.abc'); if (!fs.existsSync(cacheOutputPath) || !fs.existsSync(cacheAbcFilePath)) { logger.error(red, `ETS:ERROR ${cacheOutputPath} is lost`, reset); process.exitCode = FAIL; @@ -833,22 +831,6 @@ function writeHashJson(): void { hashJsonObject[cacheOutputPath] = hashInputContentData; hashJsonObject[cacheAbcFilePath] = hashAbcContentData; } - for (let i = 0; i < intermediateJsBundle.length; ++i) { - const abcFile: string = intermediateJsBundle[i].path.replace(/\_.js$/, ".abc"); - const cacheAbcFilePath: string = intermediateJsBundle[i].cacheOutputPath.replace(/\_.js$/, ".abc"); - if (!fs.existsSync(cacheAbcFilePath)) { - logger.error(red, `ETS:ERROR ${cacheAbcFilePath} is lost`, reset); - process.exitCode = FAIL; - break; - } - if (process.env.cachePath !== undefined) { - mkdirsSync(path.dirname(abcFile)); - fs.copyFileSync(cacheAbcFilePath, abcFile); - } - if (process.env.cachePath === undefined && fs.existsSync(intermediateJsBundle[i].path)) { - fs.unlinkSync(intermediateJsBundle[i].path); - } - } const hashFilePath: string = genHashJsonPath(buildPathInfo); if (hashFilePath.length === 0) { return; @@ -899,3 +881,32 @@ function checkNodeModules() { return true; } + +function copyFileCachePathToBuildPath() { + for (let i = 0; i < intermediateJsBundle.length; ++i) { + const abcFile: string = intermediateJsBundle[i].path.replace(/\.temp\.js$/, ".abc"); + const cacheOutputPath: string = intermediateJsBundle[i].cacheOutputPath; + const cacheAbcFilePath: string = intermediateJsBundle[i].cacheOutputPath.replace(/\.temp\.js$/, ".abc"); + if (!fs.existsSync(cacheAbcFilePath)) { + logger.error(red, `ETS:ERROR ${cacheAbcFilePath} is lost`, reset); + break; + } + let parent: string = path.join(abcFile, '..'); + if (!(fs.existsSync(parent) && fs.statSync(parent).isDirectory())) { + mkDir(parent); + } + // for preview mode, cache path and old abc file both exist, should copy abc file for updating + if (process.env.cachePath !== undefined) { + fs.copyFileSync(cacheAbcFilePath, abcFile); + } + if (process.env.cachePath === undefined && fs.existsSync(cacheOutputPath)) { + fs.unlinkSync(cacheOutputPath); + } + } +} + +function processExtraAssetForBundle() { + writeHashJson(); + copyFileCachePathToBuildPath(); + clearGlobalInfo(); +} \ No newline at end of file -- Gitee From 39286d2ef4d5eb083f5eb7650d191484ff5b0d4b Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Fri, 23 Sep 2022 14:47:38 +0800 Subject: [PATCH 2/3] Modify log level of ark & Implement different page of incre compile Signed-off-by: zhangrengao Change-Id: I0886b7f4931250ec134173080a78d1cfbe6a13be --- compiler/src/gen_abc.ts | 6 +++--- compiler/src/gen_abc_plugin.ts | 22 ++++++++++++++++------ compiler/src/gen_module_abc.ts | 6 +++--- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/compiler/src/gen_abc.ts b/compiler/src/gen_abc.ts index 83f79f8..5aab49b 100644 --- a/compiler/src/gen_abc.ts +++ b/compiler/src/gen_abc.ts @@ -39,7 +39,7 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { try { childProcess.execSync(singleCmd); } catch (e) { - logger.error(red, `ETS:ERROR Failed to convert file ${input} to abc `, reset); + logger.debug(red, `ETS:ERROR Failed to convert file ${input} to abc `, reset); process.exit(FAIL); } } @@ -58,7 +58,7 @@ function es2abcByWorkers(jsonInput: string, cmd: string): Promise { try { childProcess.execSync(singleCmd); } catch (e) { - logger.error(red, `ETS:ERROR Failed to convert file ${input} to abc `, reset); + logger.debug(red, `ETS:ERROR Failed to convert file ${input} to abc `, reset); process.exit(FAIL); } } @@ -75,7 +75,7 @@ if (cluster.isWorker && process.env['inputs'] !== undefined && process.env['cmd' } else if (process.env.panda === ES2ABC || process.env.panda === 'undefined' || process.env.panda === undefined) { es2abcByWorkers(process.env['inputs'], process.env['cmd']); } else { - logger.error(red, `ETS:ERROR please set panda module`, reset); + logger.debug(red, `ETS:ERROR please set panda module`, reset); process.exit(FAIL); } process.exit(SUCCESS); diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index 196d489..ff62499 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -186,6 +186,9 @@ export class GenAbcPlugin { if (projectConfig.compileMode === ESMODULE) { return; } + if (intermediateJsBundle.length === 0) { + return; + } buildPathInfo = output; judgeWorkersToGenAbc(invokeWorkersToGenAbc); }); @@ -364,7 +367,9 @@ function writeFileSync(inputString: string, buildPath: string, keyPath: string, } let cacheOutputPath: string = ""; if (process.env.cachePath) { - cacheOutputPath = path.join(process.env.cachePath, TEMPORARY, keyPath); + let buildDirArr: string[] = projectConfig.buildPath.split(path.sep); + let abilityDir: string = buildDirArr[buildDirArr.length - 1]; + cacheOutputPath = path.join(process.env.cachePath, TEMPORARY, abilityDir, keyPath); } else { cacheOutputPath = output; } @@ -379,7 +384,7 @@ function writeFileSync(inputString: string, buildPath: string, keyPath: string, cacheOutputPath = toUnixPath(cacheOutputPath); intermediateJsBundle.push({path: output, size: fileSize, cacheOutputPath: cacheOutputPath}); } else { - logger.error(red, `ETS:ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset); + logger.debug(red, `ETS:ERROR Failed to convert file ${jsBundleFile} to bin. ${output} is lost`, reset); process.exitCode = FAIL; } } @@ -703,7 +708,7 @@ function filterIntermediateModuleByHashJson(buildPath: string, moduleInfos: Arra const input: string = moduleInfos[i].tempFilePath; const abcPath: string = moduleInfos[i].abcFilePath; if (!fs.existsSync(input)) { - logger.error(red, `ETS:ERROR ${input} is lost`, reset); + logger.debug(red, `ETS:ERROR ${input} is lost`, reset); process.exitCode = FAIL; break; } @@ -795,7 +800,7 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil const cacheOutputPath: string = inputPaths[i].cacheOutputPath; const cacheAbcFilePath: string = cacheOutputPath.replace(/\.temp\.js$/, '.abc'); if (!fs.existsSync(cacheOutputPath)) { - logger.error(red, `ETS:ERROR ${cacheOutputPath} is lost`, reset); + logger.debug(red, `ETS:ERROR ${cacheOutputPath} is lost`, reset); process.exitCode = FAIL; break; } @@ -848,7 +853,11 @@ function genHashJsonPath(buildPath: string): string { logger.debug(red, `ETS:ERROR hash path does not exist`, reset); return ''; } - return path.join(process.env.cachePath, hashFile); + let buildDirArr: string[] = projectConfig.buildPath.split(path.sep); + let abilityDir: string = buildDirArr[buildDirArr.length - 1]; + let hashJsonPath: string = path.join(process.env.cachePath, TEMPORARY, abilityDir, hashFile); + mkdirsSync(path.dirname(hashJsonPath)); + return hashJsonPath; } else if (buildPath.indexOf(ARK) >= 0) { const dataTmps: string[] = buildPath.split(ARK); const hashPath: string = path.join(dataTmps[0], ARK); @@ -888,7 +897,8 @@ function copyFileCachePathToBuildPath() { const cacheOutputPath: string = intermediateJsBundle[i].cacheOutputPath; const cacheAbcFilePath: string = intermediateJsBundle[i].cacheOutputPath.replace(/\.temp\.js$/, ".abc"); if (!fs.existsSync(cacheAbcFilePath)) { - logger.error(red, `ETS:ERROR ${cacheAbcFilePath} is lost`, reset); + logger.debug(red, `ETS:ERROR ${cacheAbcFilePath} is lost`, reset); + process.exitCode = FAIL; break; } let parent: string = path.join(abcFile, '..'); diff --git a/compiler/src/gen_module_abc.ts b/compiler/src/gen_module_abc.ts index 1426cee..9d0ba80 100644 --- a/compiler/src/gen_module_abc.ts +++ b/compiler/src/gen_module_abc.ts @@ -47,7 +47,7 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { try { childProcess.execSync(singleCmd); } catch (e) { - logger.error(red, `ETS:ERROR Failed to convert file ${inputsStr} to abc `, reset); + logger.debug(red, `ETS:ERROR Failed to convert file ${inputsStr} to abc `, reset); process.exit(FAIL); } } @@ -65,7 +65,7 @@ function es2abcByWorkers(jsonInput: string, cmd: string): Promise { try { childProcess.execSync(singleCmd); } catch (e) { - logger.error(red, `ETS:ERROR Failed to convert file ${input} to abc `, reset); + logger.debug(red, `ETS:ERROR Failed to convert file ${input} to abc `, reset); process.exit(FAIL); } } @@ -84,7 +84,7 @@ if (cluster.isWorker && process.env['inputs'] !== undefined && process.env['cmd' } else if (process.env.panda === ES2ABC || process.env.panda === 'undefined' || process.env.panda === undefined) { es2abcByWorkers(process.env['inputs'], process.env['cmd']); } else { - logger.error(red, `ETS:ERROR please set panda module`, reset); + logger.debug(red, `ETS:ERROR please set panda module`, reset); process.exit(FAIL); } process.exit(SUCCESS); -- Gitee From 3fa677a4b86306d29ba954ce6decc55d393e4ecb Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Fri, 23 Sep 2022 14:59:17 +0800 Subject: [PATCH 3/3] Fix max listeners of preview mode Signed-off-by: zhangrengao Change-Id: I178938ef0d0e65296c052ae8098924fb6f89b1a9 --- compiler/src/gen_abc_plugin.ts | 63 ++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index ff62499..8939bb5 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -18,6 +18,7 @@ import * as path from 'path'; import cluster from 'cluster'; import process from 'process'; import os from 'os'; +import events from 'events'; import Compiler from 'webpack/lib/Compiler'; import { logger } from './compile_info'; import { @@ -62,7 +63,8 @@ let isDebug: boolean = false; let arkDir: string; let nodeJs: string; -let delayCount = 0; +let previewCount: number = 0; +let compileCount: number = 0; interface File { path: string, size: number, @@ -147,11 +149,15 @@ export class GenAbcPlugin { removeDir(projectConfig.nodeModulesPath); } + // for preview mode max listeners + events.EventEmitter.defaultMaxListeners = 100; + compiler.hooks.compilation.tap('GenAbcPlugin', (compilation) => { if (projectConfig.compileMode === JSBUNDLE || projectConfig.compileMode === undefined) { return; } buildPathInfo = output; + previewCount++; compilation.hooks.finishModules.tap('finishModules', handleFinishModules.bind(this)); }); @@ -190,7 +196,8 @@ export class GenAbcPlugin { return; } buildPathInfo = output; - judgeWorkersToGenAbc(invokeWorkersToGenAbc); + previewCount++; + invokeWorkersToGenAbc(); }); } } @@ -200,12 +207,12 @@ function clearGlobalInfo() { if (!projectConfig.isPreview) { intermediateJsBundle = []; moduleInfos = []; + entryInfos = new Map(); } fileterIntermediateJsBundle = []; filterModuleInfos = []; commonJsModuleInfos = []; ESMModuleInfos = []; - entryInfos = new Map(); hashJsonObject = {}; moduleHashJsonObject = {}; } @@ -340,7 +347,7 @@ function handleFinishModules(modules, callback): any { } }); - judgeModuleWorkersToGenAbc(invokeWorkersModuleToGenAbc); + invokeWorkersModuleToGenAbc(moduleInfos); processEntryToGenAbc(entryInfos); } @@ -513,6 +520,9 @@ function invokeCluterModuleToAbc(): void { totalWorkerNumber += esmWorkerNumber; let count_ = 0; + if (projectConfig.isPreview) { + cluster.removeAllListeners("exit"); + } cluster.on('exit', (worker, code, signal) => { if (code === FAIL || process.exitCode === FAIL) { process.exitCode = FAIL; @@ -522,8 +532,14 @@ function invokeCluterModuleToAbc(): void { if (count_ === totalWorkerNumber) { writeModuleHashJson(); clearGlobalInfo(); - if (projectConfig.isPreview) { + if (projectConfig.isPreview && compileCount < previewCount) { + compileCount++; console.info(blue, 'COMPILE RESULT:SUCCESS ', reset); + if (compileCount >= previewCount) { + return; + } + invokeWorkersModuleToGenAbc(moduleInfos); + processEntryToGenAbc(entryInfos); } } logger.debug(`worker ${worker.process.pid} finished`); @@ -583,28 +599,6 @@ function splitModulesByNumber(moduleInfos: Array, workerNumber: numb return result; } -function judgeWorkersToGenAbc(callback): void { - const workerNum: number = Object.keys(cluster.workers).length; - if (workerNum === 0) { - callback(); - return; - } else { - delayCount++; - setTimeout(judgeWorkersToGenAbc.bind(null, callback), 50); - } -} - -function judgeModuleWorkersToGenAbc(callback): void { - const workerNum: number = Object.keys(cluster.workers).length; - if (workerNum === 0) { - callback(moduleInfos); - return; - } else { - delayCount++; - setTimeout(judgeModuleWorkersToGenAbc.bind(null, callback), 50); - } -} - function invokeWorkersToGenAbc(): void { if (projectConfig.isPreview) { process.exitCode = SUCCESS; @@ -650,6 +644,10 @@ function invokeWorkersToGenAbc(): void { } let count_ = 0; + if (projectConfig.isPreview) { + process.removeAllListeners("exit"); + cluster.removeAllListeners("exit"); + } cluster.on('exit', (worker, code, signal) => { if (code === FAIL || process.exitCode === FAIL) { process.exitCode = FAIL; @@ -658,9 +656,14 @@ function invokeWorkersToGenAbc(): void { count_++; if (count_ === workerNumber) { // for preview of with incre compile - if (projectConfig.isPreview) { + if (projectConfig.isPreview && compileCount < previewCount) { + compileCount++; processExtraAssetForBundle(); console.info(red, 'COMPILE RESULT:SUCCESS ', reset); + if (compileCount >= previewCount) { + return; + } + invokeWorkersToGenAbc(); } } logger.debug(`worker ${worker.process.pid} finished`); @@ -765,7 +768,7 @@ function writeModuleHashJson(): void { return; } // fix bug of multi trigger - if (!projectConfig.isPreview || delayCount < 1) { + if (!projectConfig.isPreview || previewCount < 1) { fs.writeFileSync(hashFilePath, JSON.stringify(moduleHashJsonObject)); } } @@ -841,7 +844,7 @@ function writeHashJson(): void { return; } // fix bug of multi trigger - if (!projectConfig.isPreview || delayCount < 1) { + if (!projectConfig.isPreview || previewCount < 1) { fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); } } -- Gitee