diff --git a/compiler/main.js b/compiler/main.js index 9831b280da59b82ab1e3598d3cfc0441ac2913ed..b1f74739a38dcc3159ffae5b9ded18a679b861f0 100644 --- a/compiler/main.js +++ b/compiler/main.js @@ -315,13 +315,7 @@ function processResourceArr(resourceArr, resourceMap, filePath) { } function hashProjectPath(projectPath) { - const ASSCIIStart = 65; - const ASSCIIEnd = 90; - const deviation = 1; - process.env.hashProjectPath = - String.fromCharCode(Math.floor(Math.random() * (ASSCIIEnd - ASSCIIStart + deviation) + ASSCIIStart)) + - md5(projectPath).substring(9, 16) + - String.fromCharCode(Math.floor(Math.random() * (ASSCIIEnd - ASSCIIStart + deviation) + ASSCIIStart)); + process.env.hashProjectPath = "_" + md5(projectPath); return process.env.hashProjectPath; } diff --git a/compiler/src/gen_abc.ts b/compiler/src/gen_abc.ts index 8d20e07a55607165320f5398c414600cfc5c1b74..2c4be9d5672f8989a76794469867dacd600b089e 100644 --- a/compiler/src/gen_abc.ts +++ b/compiler/src/gen_abc.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2022 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -27,7 +27,7 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { for (let i = 0; i < inputPaths.length; ++i) { const input = inputPaths[i].path; const singleCmd = `${cmd} "${input}"`; - logger.debug("gen abc cmd is: ", singleCmd, " ,file size is:", inputPaths[i].size, " byte"); + logger.debug('gen abc cmd is: ', singleCmd, ' ,file size is:', inputPaths[i].size, ' byte'); try { childProcess.execSync(singleCmd); } catch (e) { @@ -35,10 +35,6 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { return; } - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - const abcFile: string = input.replace(/\.js$/, '.abc'); if (fs.existsSync(abcFile)) { const abcFileNew: string = abcFile.replace(/_.abc$/, '.abc'); @@ -49,10 +45,10 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { } } -logger.debug("worker data is: ", JSON.stringify(process.env)); -logger.debug("gen_abc isWorker is: ", cluster.isWorker); -if (cluster.isWorker && process.env["inputs"] !== undefined && process.env["cmd"] !== undefined) { - logger.debug("==>worker #", cluster.worker.id, "started!"); - js2abcByWorkers(process.env["inputs"], process.env["cmd"]); +logger.debug('worker data is: ', JSON.stringify(process.env)); +logger.debug('gen_abc isWorker is: ', cluster.isWorker); +if (cluster.isWorker && process.env['inputs'] !== undefined && process.env['cmd'] !== undefined) { + logger.debug('==>worker #', cluster.worker.id, 'started!'); + js2abcByWorkers(process.env['inputs'], process.env['cmd']); process.exit(); } diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index eac73f7dec773393146b565a25d5c70c411c2df8..57628d0f3d7d11f0557ca2eb51932a80b1b0cba4 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -16,9 +16,10 @@ import * as fs from 'fs'; import * as path from 'path'; import cluster from 'cluster'; -import * as process from 'process'; +import process from 'process'; import Compiler from 'webpack/lib/Compiler'; import { logger } from './compile_info'; +import { toUnixPath, toHashData } from './utils'; const firstFileEXT: string = '_.js'; const genAbcScript = 'gen_abc.js'; @@ -34,9 +35,14 @@ interface File { size: number } const intermediateJsBundle: Array = []; +let fileterIntermediateJsBundle: Array = []; +let hashJsonObject = {}; +let buildPathInfo = ''; const red: string = '\u001b[31m'; const reset: string = '\u001b[39m'; +const hashFile = 'gen_hash.json'; +const ARK = '/ark/'; export class GenAbcPlugin { constructor(output_, arkDir_, nodeJs_, isDebug_) { @@ -71,6 +77,7 @@ export class GenAbcPlugin { }); compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + buildPathInfo = output; invokeWorkersToGenAbc(); }); } @@ -107,7 +114,7 @@ function getSmallestSizeGroup(groupSize: Map) { } function splitJsBundlesBySize(bundleArray: Array, groupNumber: number) { - let result = []; + const result = []; if (bundleArray.length < groupNumber) { result.push(bundleArray); return result; @@ -146,16 +153,17 @@ function invokeWorkersToGenAbc() { js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); } + filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle); const maxWorkerNumber = 3; - const splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); + const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber); const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; const cmdPrefix: string = `${nodeJs} --expose-gc "${js2abc}" ${param} `; const clusterNewApiVersion = 16; const currentNodeVersion = parseInt(process.version.split('.')[0]); - const useNewApi = currentNodeVersion >= clusterNewApiVersion ? true : false; + const useNewApi = currentNodeVersion >= clusterNewApiVersion; - if ((useNewApi && cluster.isPrimary) || (!useNewApi && cluster.isMaster)) { + if (useNewApi && cluster.isPrimary || !useNewApi && cluster.isMaster) { if (useNewApi) { cluster.setupPrimary({ exec: path.resolve(__dirname, genAbcScript) @@ -167,15 +175,96 @@ function invokeWorkersToGenAbc() { } for (let i = 0; i < workerNumber; ++i) { - let workerData = { - "inputs": JSON.stringify(splitedBundles[i]), - "cmd": cmdPrefix - } + const workerData = { + 'inputs': JSON.stringify(splitedBundles[i]), + 'cmd': cmdPrefix + }; cluster.fork(workerData); } cluster.on('exit', (worker, code, signal) => { logger.debug(`worker ${worker.process.pid} finished`); }); + + process.on('exit', (code) => { + if (buildPathInfo.indexOf(ARK)) { + writeHashJson(); + } + }); + } +} + +function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: File[]) { + for (let i = 0; i < inputPaths.length; ++i) { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + const hashFilePath = genHashJsonPath(buildPath); + if (hashFilePath.length === 0) { + return; + } + const updateJsonObject = {}; + if (buildPath.indexOf(ARK)) { + let jsonObject = {}; + let jsonFile = ''; + if (fs.existsSync(hashFilePath)) { + jsonFile = fs.readFileSync(hashFilePath).toString(); + jsonObject = JSON.parse(jsonFile); + fileterIntermediateJsBundle = []; + for (let i = 0; i < inputPaths.length; ++i) { + const input = inputPaths[i].path; + const abcPath = input.replace(/_.js$/, '.abc'); + if (!fs.existsSync(input)) { + logger.error(red, `ETS:ERROR ${input} is lost`, reset); + continue; + } + if (fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { + updateJsonObject[input] = hashInputContentData; + updateJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } + } + } + + hashJsonObject = updateJsonObject; +} + +function writeHashJson() { + const hashFilePath = genHashJsonPath(buildPathInfo); + if (hashFilePath.length === 0) { + return; + } + for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { + const input = fileterIntermediateJsBundle[i].path; + const abcPath = input.replace(/_.js$/, '.abc'); + if (!fs.existsSync(input) || !fs.existsSync(abcPath)) { + logger.error(red, `ETS:ERROR ${input} is lost`, reset); + continue; + } + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + hashJsonObject[input] = hashInputContentData; + hashJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); + } + fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); +} + +function genHashJsonPath(buildPath: string) { + buildPath = toUnixPath(buildPath); + const dataTmps = buildPath.split(ARK); + const hashPath = path.join(dataTmps[0], ARK); + if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { + logger.error(red, `ETS:ERROR hash path does not exist`, reset); + return ''; } + return path.join(hashPath, hashFile); } diff --git a/compiler/src/utils.ts b/compiler/src/utils.ts index 59e5b9b66b3e840ac6ba0a1cde0c376b97d4d28a..7af185cc9367f30c9c3eadbba2277aef34b388a0 100644 --- a/compiler/src/utils.ts +++ b/compiler/src/utils.ts @@ -16,6 +16,7 @@ import ts from 'typescript'; import path from 'path'; import fs from 'fs'; +import { createHash } from 'crypto'; export enum LogType { ERROR = 'ERROR', @@ -206,3 +207,19 @@ export function mkDir(path_: string): void { } fs.mkdirSync(path_); } + +export function toUnixPath(data: string): string { + if (/^win/.test(require('os').platform())) { + const fileTmps: string[] = data.split(path.sep); + const newData: string = path.posix.join(...fileTmps); + return newData; + } + return data; +} + +export function toHashData(path: string) { + const content = fs.readFileSync(path); + const hash = createHash('sha256'); + hash.update(content); + return hash.digest('hex'); +}