From 5e33cce8ae3d770e613aae155bfe099c7ccda0af Mon Sep 17 00:00:00 2001 From: c00513733 Date: Sun, 13 Mar 2022 14:36:19 +0800 Subject: [PATCH] Generate abc by multi-threading Signed-off-by: c00513733 --- ace-loader/src/gen-abc.js | 42 +++++++++++++ ace-loader/src/genAbc-plugin.js | 104 +++++++++++++++++++++++++------- 2 files changed, 124 insertions(+), 22 deletions(-) create mode 100644 ace-loader/src/gen-abc.js diff --git a/ace-loader/src/gen-abc.js b/ace-loader/src/gen-abc.js new file mode 100644 index 0000000..08e6210 --- /dev/null +++ b/ace-loader/src/gen-abc.js @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('fs'); +const childProcess = require('child_process'); +const cluster = require('cluster'); + +function js2abcByWorkers(jsonInput, cmd) { + const inputPaths = JSON.parse(jsonInput); + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let singleCmd = `${cmd} "${input}"`; + childProcess.execSync(singleCmd); + + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + + const abcFile = input.replace(/\.js$/, '.abc'); + if (fs.existsSync(abcFile)) { + const abcFileNew = abcFile.replace(/_.abc$/, '.abc'); + fs.renameSync(abcFile, abcFileNew); + } + } +} + +if (cluster.isWorker && process.env["inputs"] !== undefined && process.env["cmd"] !== undefined) { + js2abcByWorkers(process.env["inputs"], process.env["cmd"]); + process.exit(); +} diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index 0cc80ac..fbb6a99 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -15,7 +15,8 @@ const fs = require('fs'); const path = require('path'); -const process = require('child_process'); +const cluster = require('cluster'); +const process = require('process'); const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' var define = globalObjects.define;' + '\n' + @@ -33,12 +34,14 @@ const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' "use strict";' + '\n'; const last = '\n' + '})(this.__appProto__);' + '\n' + '})'; const firstFileEXT = '_.js'; +const genAbcScript = 'gen-abc.js'; let output; let isWin = false; let isMac = false; let isDebug = false; let arkDir; let nodeJs; +let intermediateJsBundle = []; class GenAbcPlugin { constructor(output_, arkDir_, nodeJs_, isDebug_) { @@ -73,7 +76,10 @@ class GenAbcPlugin { writeFileSync(newContent, path.resolve(output, keyPath), key); } }) - }) + }); + compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + invokeWorkerToGenAbc(); + }); } } @@ -84,7 +90,8 @@ function writeFileSync(inputString, output, jsBundleFile) { } fs.writeFileSync(output, inputString); if (fs.existsSync(output)) { - js2abcFirst(output); + let fileSize = fs.statSync(output).size; + intermediateJsBundle.push({path: output, size: fileSize}); } } @@ -96,31 +103,84 @@ function mkDir(path_) { fs.mkdirSync(path_); } -function js2abcFirst(inputPath) { - let param = ''; - if (isDebug) { - param += '--debug'; - } +function getSmallestSizeGroup(groupSize) { + let groupSizeArray = Array.from(groupSize); + groupSizeArray.sort(function(g1, g2) { + return g1[1] - g2[1]; // sort by size + }); + return groupSizeArray[0][0]; +} - let js2abc = path.join(arkDir, 'build', 'src', 'index.js'); - if (isWin) { - js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); - } else if (isMac) { - js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); - } +function splitJsBundlesBySize(bundleArray, groupNumber) { + let result = []; + if (bundleArray.length < groupNumber) { + result.push(bundleArray); + return result; + } + + bundleArray.sort(function(f1, f2) { + return f2.size - f1.size; + }); + let groupFileSize = new Map(); + for (let i = 0; i < groupNumber; ++i) { + result.push([]); + groupFileSize.set(i, 0); + } + + let index = 0; + while(index < bundleArray.length) { + let smallestGroup = getSmallestSizeGroup(groupFileSize); + result[smallestGroup].push(bundleArray[index]); + let sizeUpdate = groupFileSize.get(smallestGroup) + bundleArray[index].size; + groupFileSize.set(smallestGroup, sizeUpdate); + index++; + } + return result; +} + +function invokeWorkerToGenAbc() { + let param = ''; + if (isDebug) { + param += ' --debug'; + } + + let js2abc = path.join(arkDir, 'build', 'src', 'index.js'); + if (isWin) { + js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); + } else if (isMac) { + js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); + } + + const maxWorkerNumber = 3; + const splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); + const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; + const cmdPrefix = `${nodeJs} --expose-gc "${js2abc}" ${param} `; - const cmd = `${nodeJs} --expose-gc "${js2abc}" "${inputPath}" ${param}`; - process.execSync(cmd); + const clusterNewApiVersion = 16; + const currentNodeVersion = parseInt(process.version.split('.')[0]); + const useNewApi = currentNodeVersion >= clusterNewApiVersion ? true : false; - if (fs.existsSync(inputPath)) { - fs.unlinkSync(inputPath); + if ((useNewApi && cluster.isPrimary) || (!useNewApi && cluster.isMaster)) { + if (useNewApi) { + cluster.setupPrimary({ + exec: path.resolve(__dirname, genAbcScript) + }); + } else { + cluster.setupMaster({ + exec: path.resolve(__dirname, genAbcScript) + }); } - let abcFile = inputPath.replace(/\.js$/, '.abc'); - if (fs.existsSync(abcFile)) { - let abcFileNew = abcFile.replace(/\_.abc$/, '.abc'); - fs.renameSync(abcFile, abcFileNew); + for (let i = 0; i < workerNumber; ++i) { + let workerData = { + "inputs": JSON.stringify(splitedBundles[i]), + "cmd": cmdPrefix + } + cluster.fork(workerData); } + + cluster.on('exit', (worker, code, signal) => {}); + } } module.exports = GenAbcPlugin; -- Gitee