From 2663c92ae94480a20703dcddb18bdbdbdb693d1c Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Mon, 14 Mar 2022 23:29:56 +0800 Subject: [PATCH 1/3] incre compile Signed-off-by: zhangrengao Change-Id: I89b12f57cc943008f8e242d33eee8ae54dd5f019 --- ace-loader/src/gen-abc.js | 4 -- ace-loader/src/genAbc-plugin.js | 100 +++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/ace-loader/src/gen-abc.js b/ace-loader/src/gen-abc.js index ba4dfc6..ef97166 100644 --- a/ace-loader/src/gen-abc.js +++ b/ace-loader/src/gen-abc.js @@ -24,10 +24,6 @@ function js2abcByWorkers(jsonInput, cmd) { let singleCmd = `${cmd} "${input}"`; childProcess.execSync(singleCmd); - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - const abcFile = input.replace(/\.js$/, '.abc'); if (fs.existsSync(abcFile)) { const abcFileNew = abcFile.replace(/_.abc$/, '.abc'); diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index fbb6a99..262b75b 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -17,6 +17,7 @@ const fs = require('fs'); const path = require('path'); const cluster = require('cluster'); const process = require('process'); +const crypto = require('crypto'); const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' var define = globalObjects.define;' + '\n' + @@ -35,6 +36,8 @@ const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + const last = '\n' + '})(this.__appProto__);' + '\n' + '})'; const firstFileEXT = '_.js'; const genAbcScript = 'gen-abc.js'; +const hashFile = 'gen_hash.json'; +const ARK = '/ark/'; let output; let isWin = false; let isMac = false; @@ -42,6 +45,9 @@ let isDebug = false; let arkDir; let nodeJs; let intermediateJsBundle = []; +let fileterIntermediateJsBundle = []; +let hashJsonObject = {}; +let buildPathInfo = ""; class GenAbcPlugin { constructor(output_, arkDir_, nodeJs_, isDebug_) { @@ -78,6 +84,7 @@ class GenAbcPlugin { }) }); compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + buildPathInfo = output; invokeWorkerToGenAbc(); }); } @@ -151,8 +158,9 @@ function invokeWorkerToGenAbc() { js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); } + filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle); const maxWorkerNumber = 3; - const splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); + const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber); const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; const cmdPrefix = `${nodeJs} --expose-gc "${js2abc}" ${param} `; @@ -180,7 +188,97 @@ function invokeWorkerToGenAbc() { } cluster.on('exit', (worker, code, signal) => {}); + + process.on('exit', (code) => { + if (buildPathInfo.indexOf(ARK)) { + const hashPath = genHashJsonPath(buildPathInfo); + const hashFilePath = path.join(hashPath, hashFile); + const parent = path.join(hashFilePath, '..'); + if (!(fs.existsSync(parent) && !fs.statSync(parent).isFile())) { + mkDir(parent); + } + for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { + let input = fileterIntermediateJsBundle[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const inputContent = fs.readFileSync(input); + const hashInput = crypto.createHash('sha256'); + hashInput.update(inputContent); + const hashInputContentData = hashInput.digest('hex'); + const abcContent = fs.readFileSync(abcPath); + const hashAbc = crypto.createHash('sha256'); + hashAbc.update(abcContent); + const hashAbcContentData = hashAbc.digest('hex'); + hashJsonObject[input] = hashInputContentData; + hashJsonObject[abcPath] = hashAbcContentData; + } + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + } + fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); + } + }); + } +} + + +function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { + for (let i = 0; i < inputPaths.length; ++i) { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + let updateJsonObject = {}; + if (buildPath.indexOf(ARK)) { + const hashPath = genHashJsonPath(buildPath); + const hashFilePath = path.join(hashPath, hashFile); + let jsonObject = {}; + let jsonFile = ""; + if (fs.existsSync(hashFilePath)) { + jsonFile = fs.readFileSync(hashFilePath).toString(); + jsonObject = JSON.parse(jsonFile); + fileterIntermediateJsBundle = []; + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const inputContent = fs.readFileSync(input); + const hashInput = crypto.createHash('sha256'); + hashInput.update(inputContent); + const hashInputContentData = hashInput.digest('hex'); + const abcContent = fs.readFileSync(abcPath); + const hashAbc = crypto.createHash('sha256'); + hashAbc.update(abcContent); + const hashAbcContentData = hashAbc.digest('hex'); + if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { + updateJsonObject[input] = hashInputContentData; + updateJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } + } + } + + hashJsonObject = updateJsonObject; +} + +function genHashJsonPath(buildPath) { + buildPath = toUnixPath(buildPath); + const dataTmps = buildPath.split(ARK); + return path.join(dataTmps[0], ARK); +} + +function toUnixPath(data) { + if (/^win/.test(require('os').platform())) { + const fileTmps = data.split(path.sep); + const newData = path.posix.join(...fileTmps); + return newData; } + return data; } module.exports = GenAbcPlugin; -- Gitee From cb6b585a8166685ddf89f79e0f69ee69c0c52523 Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Tue, 15 Mar 2022 11:05:52 +0800 Subject: [PATCH 2/3] Refactoring code Signed-off-by: zhangrengao Change-Id: I5422ddc6b826480f428ef74e1f280cc09a13592f --- ace-loader/src/genAbc-plugin.js | 77 +++++++++++++++++---------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index 262b75b..fd786ae 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -191,32 +191,7 @@ function invokeWorkerToGenAbc() { process.on('exit', (code) => { if (buildPathInfo.indexOf(ARK)) { - const hashPath = genHashJsonPath(buildPathInfo); - const hashFilePath = path.join(hashPath, hashFile); - const parent = path.join(hashFilePath, '..'); - if (!(fs.existsSync(parent) && !fs.statSync(parent).isFile())) { - mkDir(parent); - } - for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { - let input = fileterIntermediateJsBundle[i].path; - let abcPath = input.replace(/_.js$/, '.abc'); - if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const inputContent = fs.readFileSync(input); - const hashInput = crypto.createHash('sha256'); - hashInput.update(inputContent); - const hashInputContentData = hashInput.digest('hex'); - const abcContent = fs.readFileSync(abcPath); - const hashAbc = crypto.createHash('sha256'); - hashAbc.update(abcContent); - const hashAbcContentData = hashAbc.digest('hex'); - hashJsonObject[input] = hashInputContentData; - hashJsonObject[abcPath] = hashAbcContentData; - } - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - } - fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); + writeHashJson(); } }); } @@ -227,10 +202,12 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { for (let i = 0; i < inputPaths.length; ++i) { fileterIntermediateJsBundle.push(inputPaths[i]); } + const hashFilePath = genHashJsonPath(buildPath); + if (hashFilePath.length == 0) { + return ; + } let updateJsonObject = {}; if (buildPath.indexOf(ARK)) { - const hashPath = genHashJsonPath(buildPath); - const hashFilePath = path.join(hashPath, hashFile); let jsonObject = {}; let jsonFile = ""; if (fs.existsSync(hashFilePath)) { @@ -241,14 +218,8 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { let input = inputPaths[i].path; let abcPath = input.replace(/_.js$/, '.abc'); if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const inputContent = fs.readFileSync(input); - const hashInput = crypto.createHash('sha256'); - hashInput.update(inputContent); - const hashInputContentData = hashInput.digest('hex'); - const abcContent = fs.readFileSync(abcPath); - const hashAbc = crypto.createHash('sha256'); - hashAbc.update(abcContent); - const hashAbcContentData = hashAbc.digest('hex'); + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { updateJsonObject[input] = hashInputContentData; updateJsonObject[abcPath] = hashAbcContentData; @@ -266,10 +237,35 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { hashJsonObject = updateJsonObject; } +function writeHashJson() { + const hashFilePath = genHashJsonPath(buildPathInfo); + if (hashFilePath.length == 0) { + return ; + } + for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { + let input = fileterIntermediateJsBundle[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + hashJsonObject[input] = hashInputContentData; + hashJsonObject[abcPath] = hashAbcContentData; + } + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + } + fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); +} + function genHashJsonPath(buildPath) { buildPath = toUnixPath(buildPath); const dataTmps = buildPath.split(ARK); - return path.join(dataTmps[0], ARK); + const hashPath = path.join(dataTmps[0], ARK); + if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { + return ""; + } + return path.join(hashPath, hashFile); } function toUnixPath(data) { @@ -281,4 +277,11 @@ function toUnixPath(data) { return data; } +function toHashData(path) { + const content = fs.readFileSync(path); + const hash = crypto.createHash('sha256'); + hash.update(content); + return hash.digest('hex'); +} + module.exports = GenAbcPlugin; -- Gitee From a78c823d78febdb06521e6542c7c09a735221e10 Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Wed, 16 Mar 2022 21:21:52 +0800 Subject: [PATCH 3/3] fix hash path info Signed-off-by: zhangrengao Change-Id: I6b2a7625564a6516d0175056938d52e78fb748b2 --- ace-loader/src/genAbc-plugin.js | 69 ++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index fd786ae..80a47bf 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -190,9 +190,7 @@ function invokeWorkerToGenAbc() { cluster.on('exit', (worker, code, signal) => {}); process.on('exit', (code) => { - if (buildPathInfo.indexOf(ARK)) { - writeHashJson(); - } + writeHashJson(); }); } } @@ -207,29 +205,27 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { return ; } let updateJsonObject = {}; - if (buildPath.indexOf(ARK)) { - let jsonObject = {}; - let jsonFile = ""; - if (fs.existsSync(hashFilePath)) { - jsonFile = fs.readFileSync(hashFilePath).toString(); - jsonObject = JSON.parse(jsonFile); - fileterIntermediateJsBundle = []; - for (let i = 0; i < inputPaths.length; ++i) { - let input = inputPaths[i].path; - let abcPath = input.replace(/_.js$/, '.abc'); - if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const hashInputContentData = toHashData(input); - const hashAbcContentData = toHashData(abcPath); - if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { - updateJsonObject[input] = hashInputContentData; - updateJsonObject[abcPath] = hashAbcContentData; - fs.unlinkSync(input); - } else { - fileterIntermediateJsBundle.push(inputPaths[i]); - } + let jsonObject = {}; + let jsonFile = ""; + if (fs.existsSync(hashFilePath)) { + jsonFile = fs.readFileSync(hashFilePath).toString(); + jsonObject = JSON.parse(jsonFile); + fileterIntermediateJsBundle = []; + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { + updateJsonObject[input] = hashInputContentData; + updateJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); } else { fileterIntermediateJsBundle.push(inputPaths[i]); } + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); } } } @@ -238,10 +234,6 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { } function writeHashJson() { - const hashFilePath = genHashJsonPath(buildPathInfo); - if (hashFilePath.length == 0) { - return ; - } for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { let input = fileterIntermediateJsBundle[i].path; let abcPath = input.replace(/_.js$/, '.abc'); @@ -255,17 +247,30 @@ function writeHashJson() { fs.unlinkSync(input); } } + const hashFilePath = genHashJsonPath(buildPathInfo); + if (hashFilePath.length == 0) { + return ; + } fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); } function genHashJsonPath(buildPath) { buildPath = toUnixPath(buildPath); - const dataTmps = buildPath.split(ARK); - const hashPath = path.join(dataTmps[0], ARK); - if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { - return ""; + if (process.env.cachePath) { + if (!fs.existsSync(process.env.cachePath) || !fs.statSync(process.env.cachePath).isDirectory()) { + return ''; + } + return path.join(process.env.cachePath, hashFile); + } else if (buildPath.indexOf(ARK) >= 0) { + const dataTmps = buildPath.split(ARK); + const hashPath = path.join(dataTmps[0], ARK); + if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { + return ''; + } + return path.join(hashPath, hashFile); + } else { + return ''; } - return path.join(hashPath, hashFile); } function toUnixPath(data) { -- Gitee