From 17837987ee9051b534c651301bb312dd94b7b0e7 Mon Sep 17 00:00:00 2001 From: c00513733 Date: Sun, 13 Mar 2022 14:36:19 +0800 Subject: [PATCH 1/4] Generate abc by multi-threading Signed-off-by: c00513733 --- ace-loader/src/gen-abc.js | 42 +++++++++++++ ace-loader/src/genAbc-plugin.js | 104 +++++++++++++++++++++++++------- 2 files changed, 124 insertions(+), 22 deletions(-) create mode 100644 ace-loader/src/gen-abc.js diff --git a/ace-loader/src/gen-abc.js b/ace-loader/src/gen-abc.js new file mode 100644 index 0000000..ba4dfc6 --- /dev/null +++ b/ace-loader/src/gen-abc.js @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('fs'); +const childProcess = require('child_process'); +const cluster = require('cluster'); + +function js2abcByWorkers(jsonInput, cmd) { + const inputPaths = JSON.parse(jsonInput); + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let singleCmd = `${cmd} "${input}"`; + childProcess.execSync(singleCmd); + + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + + const abcFile = input.replace(/\.js$/, '.abc'); + if (fs.existsSync(abcFile)) { + const abcFileNew = abcFile.replace(/_.abc$/, '.abc'); + fs.renameSync(abcFile, abcFileNew); + } + } +} + +if (cluster.isWorker && process.env["inputs"] !== undefined && process.env["cmd"] !== undefined) { + js2abcByWorkers(process.env["inputs"], process.env["cmd"]); + process.exit(); +} diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index 0cc80ac..fbb6a99 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -15,7 +15,8 @@ const fs = require('fs'); const path = require('path'); -const process = require('child_process'); +const cluster = require('cluster'); +const process = require('process'); const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' var define = globalObjects.define;' + '\n' + @@ -33,12 +34,14 @@ const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' "use strict";' + '\n'; const last = '\n' + '})(this.__appProto__);' + '\n' + '})'; const firstFileEXT = '_.js'; +const genAbcScript = 'gen-abc.js'; let output; let isWin = false; let isMac = false; let isDebug = false; let arkDir; let nodeJs; +let intermediateJsBundle = []; class GenAbcPlugin { constructor(output_, arkDir_, nodeJs_, isDebug_) { @@ -73,7 +76,10 @@ class GenAbcPlugin { writeFileSync(newContent, path.resolve(output, keyPath), key); } }) - }) + }); + compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + invokeWorkerToGenAbc(); + }); } } @@ -84,7 +90,8 @@ function writeFileSync(inputString, output, jsBundleFile) { } fs.writeFileSync(output, inputString); if (fs.existsSync(output)) { - js2abcFirst(output); + let fileSize = fs.statSync(output).size; + intermediateJsBundle.push({path: output, size: fileSize}); } } @@ -96,31 +103,84 @@ function mkDir(path_) { fs.mkdirSync(path_); } -function js2abcFirst(inputPath) { - let param = ''; - if (isDebug) { - param += '--debug'; - } +function getSmallestSizeGroup(groupSize) { + let groupSizeArray = Array.from(groupSize); + groupSizeArray.sort(function(g1, g2) { + return g1[1] - g2[1]; // sort by size + }); + return groupSizeArray[0][0]; +} - let js2abc = path.join(arkDir, 'build', 'src', 'index.js'); - if (isWin) { - js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); - } else if (isMac) { - js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); - } +function splitJsBundlesBySize(bundleArray, groupNumber) { + let result = []; + if (bundleArray.length < groupNumber) { + result.push(bundleArray); + return result; + } + + bundleArray.sort(function(f1, f2) { + return f2.size - f1.size; + }); + let groupFileSize = new Map(); + for (let i = 0; i < groupNumber; ++i) { + result.push([]); + groupFileSize.set(i, 0); + } + + let index = 0; + while(index < bundleArray.length) { + let smallestGroup = getSmallestSizeGroup(groupFileSize); + result[smallestGroup].push(bundleArray[index]); + let sizeUpdate = groupFileSize.get(smallestGroup) + bundleArray[index].size; + groupFileSize.set(smallestGroup, sizeUpdate); + index++; + } + return result; +} + +function invokeWorkerToGenAbc() { + let param = ''; + if (isDebug) { + param += ' --debug'; + } + + let js2abc = path.join(arkDir, 'build', 'src', 'index.js'); + if (isWin) { + js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); + } else if (isMac) { + js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); + } + + const maxWorkerNumber = 3; + const splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); + const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; + const cmdPrefix = `${nodeJs} --expose-gc "${js2abc}" ${param} `; - const cmd = `${nodeJs} --expose-gc "${js2abc}" "${inputPath}" ${param}`; - process.execSync(cmd); + const clusterNewApiVersion = 16; + const currentNodeVersion = parseInt(process.version.split('.')[0]); + const useNewApi = currentNodeVersion >= clusterNewApiVersion ? true : false; - if (fs.existsSync(inputPath)) { - fs.unlinkSync(inputPath); + if ((useNewApi && cluster.isPrimary) || (!useNewApi && cluster.isMaster)) { + if (useNewApi) { + cluster.setupPrimary({ + exec: path.resolve(__dirname, genAbcScript) + }); + } else { + cluster.setupMaster({ + exec: path.resolve(__dirname, genAbcScript) + }); } - let abcFile = inputPath.replace(/\.js$/, '.abc'); - if (fs.existsSync(abcFile)) { - let abcFileNew = abcFile.replace(/\_.abc$/, '.abc'); - fs.renameSync(abcFile, abcFileNew); + for (let i = 0; i < workerNumber; ++i) { + let workerData = { + "inputs": JSON.stringify(splitedBundles[i]), + "cmd": cmdPrefix + } + cluster.fork(workerData); } + + cluster.on('exit', (worker, code, signal) => {}); + } } module.exports = GenAbcPlugin; -- Gitee From 1174472a0226a91729b107cc6c452e243d67eea8 Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Mon, 14 Mar 2022 23:29:56 +0800 Subject: [PATCH 2/4] incre compile Signed-off-by: zhangrengao Change-Id: I89b12f57cc943008f8e242d33eee8ae54dd5f019 --- ace-loader/src/gen-abc.js | 4 -- ace-loader/src/genAbc-plugin.js | 100 +++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/ace-loader/src/gen-abc.js b/ace-loader/src/gen-abc.js index ba4dfc6..ef97166 100644 --- a/ace-loader/src/gen-abc.js +++ b/ace-loader/src/gen-abc.js @@ -24,10 +24,6 @@ function js2abcByWorkers(jsonInput, cmd) { let singleCmd = `${cmd} "${input}"`; childProcess.execSync(singleCmd); - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - const abcFile = input.replace(/\.js$/, '.abc'); if (fs.existsSync(abcFile)) { const abcFileNew = abcFile.replace(/_.abc$/, '.abc'); diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index fbb6a99..262b75b 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -17,6 +17,7 @@ const fs = require('fs'); const path = require('path'); const cluster = require('cluster'); const process = require('process'); +const crypto = require('crypto'); const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' var define = globalObjects.define;' + '\n' + @@ -35,6 +36,8 @@ const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + const last = '\n' + '})(this.__appProto__);' + '\n' + '})'; const firstFileEXT = '_.js'; const genAbcScript = 'gen-abc.js'; +const hashFile = 'gen_hash.json'; +const ARK = '/ark/'; let output; let isWin = false; let isMac = false; @@ -42,6 +45,9 @@ let isDebug = false; let arkDir; let nodeJs; let intermediateJsBundle = []; +let fileterIntermediateJsBundle = []; +let hashJsonObject = {}; +let buildPathInfo = ""; class GenAbcPlugin { constructor(output_, arkDir_, nodeJs_, isDebug_) { @@ -78,6 +84,7 @@ class GenAbcPlugin { }) }); compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + buildPathInfo = output; invokeWorkerToGenAbc(); }); } @@ -151,8 +158,9 @@ function invokeWorkerToGenAbc() { js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); } + filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle); const maxWorkerNumber = 3; - const splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); + const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber); const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; const cmdPrefix = `${nodeJs} --expose-gc "${js2abc}" ${param} `; @@ -180,7 +188,97 @@ function invokeWorkerToGenAbc() { } cluster.on('exit', (worker, code, signal) => {}); + + process.on('exit', (code) => { + if (buildPathInfo.indexOf(ARK)) { + const hashPath = genHashJsonPath(buildPathInfo); + const hashFilePath = path.join(hashPath, hashFile); + const parent = path.join(hashFilePath, '..'); + if (!(fs.existsSync(parent) && !fs.statSync(parent).isFile())) { + mkDir(parent); + } + for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { + let input = fileterIntermediateJsBundle[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const inputContent = fs.readFileSync(input); + const hashInput = crypto.createHash('sha256'); + hashInput.update(inputContent); + const hashInputContentData = hashInput.digest('hex'); + const abcContent = fs.readFileSync(abcPath); + const hashAbc = crypto.createHash('sha256'); + hashAbc.update(abcContent); + const hashAbcContentData = hashAbc.digest('hex'); + hashJsonObject[input] = hashInputContentData; + hashJsonObject[abcPath] = hashAbcContentData; + } + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + } + fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); + } + }); + } +} + + +function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { + for (let i = 0; i < inputPaths.length; ++i) { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + let updateJsonObject = {}; + if (buildPath.indexOf(ARK)) { + const hashPath = genHashJsonPath(buildPath); + const hashFilePath = path.join(hashPath, hashFile); + let jsonObject = {}; + let jsonFile = ""; + if (fs.existsSync(hashFilePath)) { + jsonFile = fs.readFileSync(hashFilePath).toString(); + jsonObject = JSON.parse(jsonFile); + fileterIntermediateJsBundle = []; + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const inputContent = fs.readFileSync(input); + const hashInput = crypto.createHash('sha256'); + hashInput.update(inputContent); + const hashInputContentData = hashInput.digest('hex'); + const abcContent = fs.readFileSync(abcPath); + const hashAbc = crypto.createHash('sha256'); + hashAbc.update(abcContent); + const hashAbcContentData = hashAbc.digest('hex'); + if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { + updateJsonObject[input] = hashInputContentData; + updateJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } + } + } + + hashJsonObject = updateJsonObject; +} + +function genHashJsonPath(buildPath) { + buildPath = toUnixPath(buildPath); + const dataTmps = buildPath.split(ARK); + return path.join(dataTmps[0], ARK); +} + +function toUnixPath(data) { + if (/^win/.test(require('os').platform())) { + const fileTmps = data.split(path.sep); + const newData = path.posix.join(...fileTmps); + return newData; } + return data; } module.exports = GenAbcPlugin; -- Gitee From 0fc66771a9366be43e6c0cee6e9f1e3b9fe0005d Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Tue, 15 Mar 2022 11:05:52 +0800 Subject: [PATCH 3/4] Refactoring code Signed-off-by: zhangrengao Change-Id: I5422ddc6b826480f428ef74e1f280cc09a13592f --- ace-loader/src/genAbc-plugin.js | 77 +++++++++++++++++---------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index 262b75b..fd786ae 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -191,32 +191,7 @@ function invokeWorkerToGenAbc() { process.on('exit', (code) => { if (buildPathInfo.indexOf(ARK)) { - const hashPath = genHashJsonPath(buildPathInfo); - const hashFilePath = path.join(hashPath, hashFile); - const parent = path.join(hashFilePath, '..'); - if (!(fs.existsSync(parent) && !fs.statSync(parent).isFile())) { - mkDir(parent); - } - for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { - let input = fileterIntermediateJsBundle[i].path; - let abcPath = input.replace(/_.js$/, '.abc'); - if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const inputContent = fs.readFileSync(input); - const hashInput = crypto.createHash('sha256'); - hashInput.update(inputContent); - const hashInputContentData = hashInput.digest('hex'); - const abcContent = fs.readFileSync(abcPath); - const hashAbc = crypto.createHash('sha256'); - hashAbc.update(abcContent); - const hashAbcContentData = hashAbc.digest('hex'); - hashJsonObject[input] = hashInputContentData; - hashJsonObject[abcPath] = hashAbcContentData; - } - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - } - fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); + writeHashJson(); } }); } @@ -227,10 +202,12 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { for (let i = 0; i < inputPaths.length; ++i) { fileterIntermediateJsBundle.push(inputPaths[i]); } + const hashFilePath = genHashJsonPath(buildPath); + if (hashFilePath.length == 0) { + return ; + } let updateJsonObject = {}; if (buildPath.indexOf(ARK)) { - const hashPath = genHashJsonPath(buildPath); - const hashFilePath = path.join(hashPath, hashFile); let jsonObject = {}; let jsonFile = ""; if (fs.existsSync(hashFilePath)) { @@ -241,14 +218,8 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { let input = inputPaths[i].path; let abcPath = input.replace(/_.js$/, '.abc'); if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const inputContent = fs.readFileSync(input); - const hashInput = crypto.createHash('sha256'); - hashInput.update(inputContent); - const hashInputContentData = hashInput.digest('hex'); - const abcContent = fs.readFileSync(abcPath); - const hashAbc = crypto.createHash('sha256'); - hashAbc.update(abcContent); - const hashAbcContentData = hashAbc.digest('hex'); + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { updateJsonObject[input] = hashInputContentData; updateJsonObject[abcPath] = hashAbcContentData; @@ -266,10 +237,35 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { hashJsonObject = updateJsonObject; } +function writeHashJson() { + const hashFilePath = genHashJsonPath(buildPathInfo); + if (hashFilePath.length == 0) { + return ; + } + for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { + let input = fileterIntermediateJsBundle[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + hashJsonObject[input] = hashInputContentData; + hashJsonObject[abcPath] = hashAbcContentData; + } + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + } + fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); +} + function genHashJsonPath(buildPath) { buildPath = toUnixPath(buildPath); const dataTmps = buildPath.split(ARK); - return path.join(dataTmps[0], ARK); + const hashPath = path.join(dataTmps[0], ARK); + if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { + return ""; + } + return path.join(hashPath, hashFile); } function toUnixPath(data) { @@ -281,4 +277,11 @@ function toUnixPath(data) { return data; } +function toHashData(path) { + const content = fs.readFileSync(path); + const hash = crypto.createHash('sha256'); + hash.update(content); + return hash.digest('hex'); +} + module.exports = GenAbcPlugin; -- Gitee From 54cad8fbe910418f5e561f943548ee9fe54b4825 Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Wed, 16 Mar 2022 21:21:52 +0800 Subject: [PATCH 4/4] fix hash path info Signed-off-by: zhangrengao Change-Id: I6b2a7625564a6516d0175056938d52e78fb748b2 --- ace-loader/src/genAbc-plugin.js | 69 ++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index fd786ae..80a47bf 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -190,9 +190,7 @@ function invokeWorkerToGenAbc() { cluster.on('exit', (worker, code, signal) => {}); process.on('exit', (code) => { - if (buildPathInfo.indexOf(ARK)) { - writeHashJson(); - } + writeHashJson(); }); } } @@ -207,29 +205,27 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { return ; } let updateJsonObject = {}; - if (buildPath.indexOf(ARK)) { - let jsonObject = {}; - let jsonFile = ""; - if (fs.existsSync(hashFilePath)) { - jsonFile = fs.readFileSync(hashFilePath).toString(); - jsonObject = JSON.parse(jsonFile); - fileterIntermediateJsBundle = []; - for (let i = 0; i < inputPaths.length; ++i) { - let input = inputPaths[i].path; - let abcPath = input.replace(/_.js$/, '.abc'); - if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const hashInputContentData = toHashData(input); - const hashAbcContentData = toHashData(abcPath); - if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { - updateJsonObject[input] = hashInputContentData; - updateJsonObject[abcPath] = hashAbcContentData; - fs.unlinkSync(input); - } else { - fileterIntermediateJsBundle.push(inputPaths[i]); - } + let jsonObject = {}; + let jsonFile = ""; + if (fs.existsSync(hashFilePath)) { + jsonFile = fs.readFileSync(hashFilePath).toString(); + jsonObject = JSON.parse(jsonFile); + fileterIntermediateJsBundle = []; + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { + updateJsonObject[input] = hashInputContentData; + updateJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); } else { fileterIntermediateJsBundle.push(inputPaths[i]); } + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); } } } @@ -238,10 +234,6 @@ function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { } function writeHashJson() { - const hashFilePath = genHashJsonPath(buildPathInfo); - if (hashFilePath.length == 0) { - return ; - } for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { let input = fileterIntermediateJsBundle[i].path; let abcPath = input.replace(/_.js$/, '.abc'); @@ -255,17 +247,30 @@ function writeHashJson() { fs.unlinkSync(input); } } + const hashFilePath = genHashJsonPath(buildPathInfo); + if (hashFilePath.length == 0) { + return ; + } fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); } function genHashJsonPath(buildPath) { buildPath = toUnixPath(buildPath); - const dataTmps = buildPath.split(ARK); - const hashPath = path.join(dataTmps[0], ARK); - if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { - return ""; + if (process.env.cachePath) { + if (!fs.existsSync(process.env.cachePath) || !fs.statSync(process.env.cachePath).isDirectory()) { + return ''; + } + return path.join(process.env.cachePath, hashFile); + } else if (buildPath.indexOf(ARK) >= 0) { + const dataTmps = buildPath.split(ARK); + const hashPath = path.join(dataTmps[0], ARK); + if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { + return ''; + } + return path.join(hashPath, hashFile); + } else { + return ''; } - return path.join(hashPath, hashFile); } function toUnixPath(data) { -- Gitee