diff --git a/ace-loader/src/gen-abc.js b/ace-loader/src/gen-abc.js new file mode 100644 index 0000000000000000000000000000000000000000..ef97166079b8ac46dd4312a2e2b54a685bd21311 --- /dev/null +++ b/ace-loader/src/gen-abc.js @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fs = require('fs'); +const childProcess = require('child_process'); +const cluster = require('cluster'); + +function js2abcByWorkers(jsonInput, cmd) { + const inputPaths = JSON.parse(jsonInput); + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let singleCmd = `${cmd} "${input}"`; + childProcess.execSync(singleCmd); + + const abcFile = input.replace(/\.js$/, '.abc'); + if (fs.existsSync(abcFile)) { + const abcFileNew = abcFile.replace(/_.abc$/, '.abc'); + fs.renameSync(abcFile, abcFileNew); + } + } +} + +if (cluster.isWorker && process.env["inputs"] !== undefined && process.env["cmd"] !== undefined) { + js2abcByWorkers(process.env["inputs"], process.env["cmd"]); + process.exit(); +} diff --git a/ace-loader/src/genAbc-plugin.js b/ace-loader/src/genAbc-plugin.js index 0cc80ac8cd3c2bac44ddb886253b1c5d35de01e0..80a47bf387fdcb9ac7f5d4fee2e36684e15942a8 100644 --- a/ace-loader/src/genAbc-plugin.js +++ b/ace-loader/src/genAbc-plugin.js @@ -15,7 +15,9 @@ const fs = require('fs'); const path = require('path'); -const process = require('child_process'); +const cluster = require('cluster'); +const process = require('process'); +const crypto = require('crypto'); const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' var define = globalObjects.define;' + '\n' + @@ -33,12 +35,19 @@ const forward = '(global.___mainEntry___ = function (globalObjects) {' + '\n' + ' "use strict";' + '\n'; const last = '\n' + '})(this.__appProto__);' + '\n' + '})'; const firstFileEXT = '_.js'; +const genAbcScript = 'gen-abc.js'; +const hashFile = 'gen_hash.json'; +const ARK = '/ark/'; let output; let isWin = false; let isMac = false; let isDebug = false; let arkDir; let nodeJs; +let intermediateJsBundle = []; +let fileterIntermediateJsBundle = []; +let hashJsonObject = {}; +let buildPathInfo = ""; class GenAbcPlugin { constructor(output_, arkDir_, nodeJs_, isDebug_) { @@ -73,7 +82,11 @@ class GenAbcPlugin { writeFileSync(newContent, path.resolve(output, keyPath), key); } }) - }) + }); + compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + buildPathInfo = output; + invokeWorkerToGenAbc(); + }); } } @@ -84,7 +97,8 @@ function writeFileSync(inputString, output, jsBundleFile) { } fs.writeFileSync(output, inputString); if (fs.existsSync(output)) { - js2abcFirst(output); + let fileSize = fs.statSync(output).size; + intermediateJsBundle.push({path: output, size: fileSize}); } } @@ -96,31 +110,183 @@ function mkDir(path_) { fs.mkdirSync(path_); } -function js2abcFirst(inputPath) { - let param = ''; - if (isDebug) { - param += '--debug'; +function getSmallestSizeGroup(groupSize) { + let groupSizeArray = Array.from(groupSize); + groupSizeArray.sort(function(g1, g2) { + return g1[1] - g2[1]; // sort by size + }); + return groupSizeArray[0][0]; +} + +function splitJsBundlesBySize(bundleArray, groupNumber) { + let result = []; + if (bundleArray.length < groupNumber) { + result.push(bundleArray); + return result; + } + + bundleArray.sort(function(f1, f2) { + return f2.size - f1.size; + }); + let groupFileSize = new Map(); + for (let i = 0; i < groupNumber; ++i) { + result.push([]); + groupFileSize.set(i, 0); + } + + let index = 0; + while(index < bundleArray.length) { + let smallestGroup = getSmallestSizeGroup(groupFileSize); + result[smallestGroup].push(bundleArray[index]); + let sizeUpdate = groupFileSize.get(smallestGroup) + bundleArray[index].size; + groupFileSize.set(smallestGroup, sizeUpdate); + index++; + } + return result; +} + +function invokeWorkerToGenAbc() { + let param = ''; + if (isDebug) { + param += ' --debug'; + } + + let js2abc = path.join(arkDir, 'build', 'src', 'index.js'); + if (isWin) { + js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); + } else if (isMac) { + js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); + } + + filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle); + const maxWorkerNumber = 3; + const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber); + const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; + const cmdPrefix = `${nodeJs} --expose-gc "${js2abc}" ${param} `; + + const clusterNewApiVersion = 16; + const currentNodeVersion = parseInt(process.version.split('.')[0]); + const useNewApi = currentNodeVersion >= clusterNewApiVersion ? true : false; + + if ((useNewApi && cluster.isPrimary) || (!useNewApi && cluster.isMaster)) { + if (useNewApi) { + cluster.setupPrimary({ + exec: path.resolve(__dirname, genAbcScript) + }); + } else { + cluster.setupMaster({ + exec: path.resolve(__dirname, genAbcScript) + }); } - let js2abc = path.join(arkDir, 'build', 'src', 'index.js'); - if (isWin) { - js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); - } else if (isMac) { - js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); + for (let i = 0; i < workerNumber; ++i) { + let workerData = { + "inputs": JSON.stringify(splitedBundles[i]), + "cmd": cmdPrefix + } + cluster.fork(workerData); } - const cmd = `${nodeJs} --expose-gc "${js2abc}" "${inputPath}" ${param}`; - process.execSync(cmd); + cluster.on('exit', (worker, code, signal) => {}); + + process.on('exit', (code) => { + writeHashJson(); + }); + } +} + - if (fs.existsSync(inputPath)) { - fs.unlinkSync(inputPath); +function filterIntermediateJsBundleByHashJson(buildPath, inputPaths) { + for (let i = 0; i < inputPaths.length; ++i) { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + const hashFilePath = genHashJsonPath(buildPath); + if (hashFilePath.length == 0) { + return ; + } + let updateJsonObject = {}; + let jsonObject = {}; + let jsonFile = ""; + if (fs.existsSync(hashFilePath)) { + jsonFile = fs.readFileSync(hashFilePath).toString(); + jsonObject = JSON.parse(jsonFile); + fileterIntermediateJsBundle = []; + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { + updateJsonObject[input] = hashInputContentData; + updateJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } } + } + + hashJsonObject = updateJsonObject; +} + +function writeHashJson() { + for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { + let input = fileterIntermediateJsBundle[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + hashJsonObject[input] = hashInputContentData; + hashJsonObject[abcPath] = hashAbcContentData; + } + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + } + const hashFilePath = genHashJsonPath(buildPathInfo); + if (hashFilePath.length == 0) { + return ; + } + fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); +} - let abcFile = inputPath.replace(/\.js$/, '.abc'); - if (fs.existsSync(abcFile)) { - let abcFileNew = abcFile.replace(/\_.abc$/, '.abc'); - fs.renameSync(abcFile, abcFileNew); +function genHashJsonPath(buildPath) { + buildPath = toUnixPath(buildPath); + if (process.env.cachePath) { + if (!fs.existsSync(process.env.cachePath) || !fs.statSync(process.env.cachePath).isDirectory()) { + return ''; + } + return path.join(process.env.cachePath, hashFile); + } else if (buildPath.indexOf(ARK) >= 0) { + const dataTmps = buildPath.split(ARK); + const hashPath = path.join(dataTmps[0], ARK); + if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { + return ''; } + return path.join(hashPath, hashFile); + } else { + return ''; + } +} + +function toUnixPath(data) { + if (/^win/.test(require('os').platform())) { + const fileTmps = data.split(path.sep); + const newData = path.posix.join(...fileTmps); + return newData; + } + return data; +} + +function toHashData(path) { + const content = fs.readFileSync(path); + const hash = crypto.createHash('sha256'); + hash.update(content); + return hash.digest('hex'); } module.exports = GenAbcPlugin;