From 945ce934ab12887654b402d6d10a864354d4603d Mon Sep 17 00:00:00 2001 From: c00513733 Date: Sun, 13 Mar 2022 15:24:06 +0800 Subject: [PATCH 1/4] Generate abc by multi-threading Signed-off-by: c00513733 Change-Id: Ib21e5fc5e380586314e4fa238a2ea7022dfa81d8 --- compiler/src/gen_abc.ts | 42 +++++++++++------ compiler/src/gen_abc_plugin.ts | 82 ++++++++++++++++++++++------------ 2 files changed, 81 insertions(+), 43 deletions(-) diff --git a/compiler/src/gen_abc.ts b/compiler/src/gen_abc.ts index 8bcd6ff33..8d20e07a5 100644 --- a/compiler/src/gen_abc.ts +++ b/compiler/src/gen_abc.ts @@ -1,23 +1,35 @@ -import * as process from 'child_process'; +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as childProcess from 'child_process'; +import * as process from 'process'; import * as fs from 'fs'; +import cluster from 'cluster'; import { logger } from './compile_info'; -const {workerData, threadId} = require('worker_threads'); - -interface File { - path: string, - size: number -} const red: string = '\u001b[31m'; const reset: string = '\u001b[39m'; -function js2abcByWorkers(inputPaths: File[], cmd: string): Promise { +function js2abcByWorkers(jsonInput: string, cmd: string): Promise { + const inputPaths = JSON.parse(jsonInput); for (let i = 0; i < inputPaths.length; ++i) { const input = inputPaths[i].path; const singleCmd = `${cmd} "${input}"`; - logger.debug('gen abc cmd is: ', singleCmd); + logger.debug("gen abc cmd is: ", singleCmd, " ,file size is:", inputPaths[i].size, " byte"); try { - process.execSync(singleCmd); + childProcess.execSync(singleCmd); } catch (e) { logger.error(red, `ETS:ERROR Failed to convert file ${input} to abc `, reset); return; @@ -37,8 +49,10 @@ function js2abcByWorkers(inputPaths: File[], cmd: string): Promise { } } -logger.debug('worker data is: ', JSON.stringify(workerData)); -if (JSON.stringify(workerData) !== 'null') { - logger.debug('==>worker #', threadId, 'started!'); - js2abcByWorkers(workerData.input, workerData.cmd); +logger.debug("worker data is: ", JSON.stringify(process.env)); +logger.debug("gen_abc isWorker is: ", cluster.isWorker); +if (cluster.isWorker && process.env["inputs"] !== undefined && process.env["cmd"] !== undefined) { + logger.debug("==>worker #", cluster.worker.id, "started!"); + js2abcByWorkers(process.env["inputs"], process.env["cmd"]); + process.exit(); } diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index 3de104e3e..eac73f7de 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -15,10 +15,11 @@ import * as fs from 'fs'; import * as path from 'path'; +import cluster from 'cluster'; +import * as process from 'process'; import Compiler from 'webpack/lib/Compiler'; import { logger } from './compile_info'; -const {Worker, isMainThread} = require('worker_threads'); const firstFileEXT: string = '_.js'; const genAbcScript = 'gen_abc.js'; let output: string; @@ -70,31 +71,7 @@ export class GenAbcPlugin { }); compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { - let param: string = ''; - if (isDebug) { - param += ' --debug'; - } - - if (isMainThread) { - let js2abc: string = path.join(arkDir, 'build', 'src', 'index.js'); - if (isWin) { - js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); - } else if (isMac) { - js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); - } - const maxWorkerNumber = 3; - const splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); - const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; - const cmdPrefix: string = `${nodeJs} --expose-gc "${js2abc}" ${param} `; - const workers = []; - for (let i = 0; i < workerNumber; ++i) { - workers.push(new Worker(path.resolve(__dirname, genAbcScript), - {workerData: {input: splitedBundles[i], cmd: cmdPrefix} })); - workers[i].on('exit', () => { - logger.debug('worker ', i, 'finished!'); - }); - } - } + invokeWorkersToGenAbc(); }); } } @@ -124,13 +101,13 @@ function mkDir(path_: string): void { function getSmallestSizeGroup(groupSize: Map) { const groupSizeArray = Array.from(groupSize); groupSizeArray.sort(function(g1, g2) { - return g1[1] - g2[1]; // sort by value + return g1[1] - g2[1]; // sort by size }); - return groupSizeArray[0][0]; // return key + return groupSizeArray[0][0]; } function splitJsBundlesBySize(bundleArray: Array, groupNumber: number) { - const result = []; + let result = []; if (bundleArray.length < groupNumber) { result.push(bundleArray); return result; @@ -155,3 +132,50 @@ function splitJsBundlesBySize(bundleArray: Array, groupNumber: number) { } return result; } + +function invokeWorkersToGenAbc() { + let param: string = ''; + if (isDebug) { + param += ' --debug'; + } + + let js2abc: string = path.join(arkDir, 'build', 'src', 'index.js'); + if (isWin) { + js2abc = path.join(arkDir, 'build-win', 'src', 'index.js'); + } else if (isMac) { + js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); + } + + const maxWorkerNumber = 3; + const splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); + const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; + const cmdPrefix: string = `${nodeJs} --expose-gc "${js2abc}" ${param} `; + + const clusterNewApiVersion = 16; + const currentNodeVersion = parseInt(process.version.split('.')[0]); + const useNewApi = currentNodeVersion >= clusterNewApiVersion ? true : false; + + if ((useNewApi && cluster.isPrimary) || (!useNewApi && cluster.isMaster)) { + if (useNewApi) { + cluster.setupPrimary({ + exec: path.resolve(__dirname, genAbcScript) + }); + } else { + cluster.setupMaster({ + exec: path.resolve(__dirname, genAbcScript) + }); + } + + for (let i = 0; i < workerNumber; ++i) { + let workerData = { + "inputs": JSON.stringify(splitedBundles[i]), + "cmd": cmdPrefix + } + cluster.fork(workerData); + } + + cluster.on('exit', (worker, code, signal) => { + logger.debug(`worker ${worker.process.pid} finished`); + }); + } +} -- Gitee From fd48684c24f669173f2c5260345b0c394deeb9c9 Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Mon, 14 Mar 2022 15:04:57 +0800 Subject: [PATCH 2/4] incre compiler abc Signed-off-by: zhangrengao Change-Id: Ieeeffa9f63988167726789faeee7528b2ffce0ba --- compiler/src/gen_abc.ts | 6 +-- compiler/src/gen_abc_plugin.ts | 95 ++++++++++++++++++++++++++++++++-- compiler/src/utils.ts | 9 ++++ 3 files changed, 102 insertions(+), 8 deletions(-) diff --git a/compiler/src/gen_abc.ts b/compiler/src/gen_abc.ts index 8d20e07a5..dde95b102 100644 --- a/compiler/src/gen_abc.ts +++ b/compiler/src/gen_abc.ts @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Huawei Device Co., Ltd. + * Copyright (c) 2022 Huawei Device Co., Ltd. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -35,10 +35,6 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { return; } - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - const abcFile: string = input.replace(/\.js$/, '.abc'); if (fs.existsSync(abcFile)) { const abcFileNew: string = abcFile.replace(/_.abc$/, '.abc'); diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index eac73f7de..f0b840dd9 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -16,9 +16,11 @@ import * as fs from 'fs'; import * as path from 'path'; import cluster from 'cluster'; -import * as process from 'process'; +import process from 'process'; import Compiler from 'webpack/lib/Compiler'; import { logger } from './compile_info'; +import { toUnixPath } from './utils'; +import { createHash } from 'crypto'; const firstFileEXT: string = '_.js'; const genAbcScript = 'gen_abc.js'; @@ -33,10 +35,15 @@ interface File { path: string, size: number } -const intermediateJsBundle: Array = []; +let intermediateJsBundle: Array = []; +let fileterIntermediateJsBundle: Array = []; +let hashJsonObject = {}; +let buildPathInfo = ""; const red: string = '\u001b[31m'; const reset: string = '\u001b[39m'; +const hashFile = 'gen_hash.json'; +const ARK = '/ark/'; export class GenAbcPlugin { constructor(output_, arkDir_, nodeJs_, isDebug_) { @@ -71,6 +78,7 @@ export class GenAbcPlugin { }); compiler.hooks.afterEmit.tap('GenAbcPluginMultiThread', () => { + buildPathInfo = output; invokeWorkersToGenAbc(); }); } @@ -146,8 +154,9 @@ function invokeWorkersToGenAbc() { js2abc = path.join(arkDir, 'build-mac', 'src', 'index.js'); } + filterIntermediateJsBundleByHashJson(buildPathInfo, intermediateJsBundle); const maxWorkerNumber = 3; - const splitedBundles = splitJsBundlesBySize(intermediateJsBundle, maxWorkerNumber); + const splitedBundles = splitJsBundlesBySize(fileterIntermediateJsBundle, maxWorkerNumber); const workerNumber = maxWorkerNumber < splitedBundles.length ? maxWorkerNumber : splitedBundles.length; const cmdPrefix: string = `${nodeJs} --expose-gc "${js2abc}" ${param} `; @@ -177,5 +186,85 @@ function invokeWorkersToGenAbc() { cluster.on('exit', (worker, code, signal) => { logger.debug(`worker ${worker.process.pid} finished`); }); + + process.on('exit', (code) => { + if (buildPathInfo.indexOf(ARK)) { + const hashPath = genHashJsonPath(buildPathInfo); + const hashFilePath = path.join(hashPath, hashFile); + const parent: string = path.join(hashFilePath, '..'); + if (!(fs.existsSync(parent) && !fs.statSync(parent).isFile())) { + mkDir(parent); + } + for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { + let input = fileterIntermediateJsBundle[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const inputContent = fs.readFileSync(input); + const hashInput = createHash('sha256'); + hashInput.update(inputContent); + const hashInputContentData = hashInput.digest('hex'); + const abcContent = fs.readFileSync(abcPath); + const hashAbc = createHash('sha256'); + hashAbc.update(abcContent); + const hashAbcContentData = hashAbc.digest('hex'); + hashJsonObject[input] = hashInputContentData; + hashJsonObject[abcPath] = hashAbcContentData; + } + if (fs.existsSync(input)) { + fs.unlinkSync(input); + } + } + fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); + } + }) + } +} + +function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: File[]) { + for (let i = 0; i < inputPaths.length; ++i) { + fileterIntermediateJsBundle.push(inputPaths[i]); } + let updateJsonObject = {}; + if (buildPath.indexOf(ARK)) { + const hashPath = genHashJsonPath(buildPath); + const hashFilePath = path.join(hashPath, hashFile); + let jsonObject = {}; + let jsonFile = ""; + if (fs.existsSync(hashFilePath)) { + jsonFile = fs.readFileSync(hashFilePath).toString(); + jsonObject = JSON.parse(jsonFile); + fileterIntermediateJsBundle = []; + for (let i = 0; i < inputPaths.length; ++i) { + let input = inputPaths[i].path; + let abcPath = input.replace(/_.js$/, '.abc'); + if (fs.existsSync(input) && fs.existsSync(abcPath)) { + const inputContent = fs.readFileSync(input); + const hashInput = createHash('sha256'); + hashInput.update(inputContent); + const hashInputContentData = hashInput.digest('hex'); + const abcContent = fs.readFileSync(abcPath); + const hashAbc = createHash('sha256'); + hashAbc.update(abcContent); + const hashAbcContentData = hashAbc.digest('hex'); + if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { + updateJsonObject[input] = hashInputContentData; + updateJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); + } + } + } + } + + hashJsonObject = updateJsonObject; +} + +function genHashJsonPath(buildPath: string) { + buildPath = toUnixPath(buildPath); + const dataTmps = buildPath.split(ARK); + return path.join(dataTmps[0], ARK); } diff --git a/compiler/src/utils.ts b/compiler/src/utils.ts index 59e5b9b66..d491a4148 100644 --- a/compiler/src/utils.ts +++ b/compiler/src/utils.ts @@ -206,3 +206,12 @@ export function mkDir(path_: string): void { } fs.mkdirSync(path_); } + +export function toUnixPath(data: string): string { + if (/^win/.test(require('os').platform())) { + const fileTmps: string[] = data.split(path.sep); + const newData: string = path.posix.join(...fileTmps); + return newData; + } + return data; +} \ No newline at end of file -- Gitee From abf66174394c76dbad60b9aef6ed201bfc73ed12 Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Tue, 15 Mar 2022 10:49:58 +0800 Subject: [PATCH 3/4] Refactoring code Signed-off-by: zhangrengao Change-Id: I568f4e6555be8898b40282c2ad4a09a90ded9dc6 --- compiler/src/gen_abc.ts | 12 ++-- compiler/src/gen_abc_plugin.ts | 108 ++++++++++++++++----------------- compiler/src/utils.ts | 10 ++- 3 files changed, 69 insertions(+), 61 deletions(-) diff --git a/compiler/src/gen_abc.ts b/compiler/src/gen_abc.ts index dde95b102..2c4be9d56 100644 --- a/compiler/src/gen_abc.ts +++ b/compiler/src/gen_abc.ts @@ -27,7 +27,7 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { for (let i = 0; i < inputPaths.length; ++i) { const input = inputPaths[i].path; const singleCmd = `${cmd} "${input}"`; - logger.debug("gen abc cmd is: ", singleCmd, " ,file size is:", inputPaths[i].size, " byte"); + logger.debug('gen abc cmd is: ', singleCmd, ' ,file size is:', inputPaths[i].size, ' byte'); try { childProcess.execSync(singleCmd); } catch (e) { @@ -45,10 +45,10 @@ function js2abcByWorkers(jsonInput: string, cmd: string): Promise { } } -logger.debug("worker data is: ", JSON.stringify(process.env)); -logger.debug("gen_abc isWorker is: ", cluster.isWorker); -if (cluster.isWorker && process.env["inputs"] !== undefined && process.env["cmd"] !== undefined) { - logger.debug("==>worker #", cluster.worker.id, "started!"); - js2abcByWorkers(process.env["inputs"], process.env["cmd"]); +logger.debug('worker data is: ', JSON.stringify(process.env)); +logger.debug('gen_abc isWorker is: ', cluster.isWorker); +if (cluster.isWorker && process.env['inputs'] !== undefined && process.env['cmd'] !== undefined) { + logger.debug('==>worker #', cluster.worker.id, 'started!'); + js2abcByWorkers(process.env['inputs'], process.env['cmd']); process.exit(); } diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index f0b840dd9..57628d0f3 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -19,8 +19,7 @@ import cluster from 'cluster'; import process from 'process'; import Compiler from 'webpack/lib/Compiler'; import { logger } from './compile_info'; -import { toUnixPath } from './utils'; -import { createHash } from 'crypto'; +import { toUnixPath, toHashData } from './utils'; const firstFileEXT: string = '_.js'; const genAbcScript = 'gen_abc.js'; @@ -35,10 +34,10 @@ interface File { path: string, size: number } -let intermediateJsBundle: Array = []; +const intermediateJsBundle: Array = []; let fileterIntermediateJsBundle: Array = []; let hashJsonObject = {}; -let buildPathInfo = ""; +let buildPathInfo = ''; const red: string = '\u001b[31m'; const reset: string = '\u001b[39m'; @@ -115,7 +114,7 @@ function getSmallestSizeGroup(groupSize: Map) { } function splitJsBundlesBySize(bundleArray: Array, groupNumber: number) { - let result = []; + const result = []; if (bundleArray.length < groupNumber) { result.push(bundleArray); return result; @@ -162,9 +161,9 @@ function invokeWorkersToGenAbc() { const clusterNewApiVersion = 16; const currentNodeVersion = parseInt(process.version.split('.')[0]); - const useNewApi = currentNodeVersion >= clusterNewApiVersion ? true : false; + const useNewApi = currentNodeVersion >= clusterNewApiVersion; - if ((useNewApi && cluster.isPrimary) || (!useNewApi && cluster.isMaster)) { + if (useNewApi && cluster.isPrimary || !useNewApi && cluster.isMaster) { if (useNewApi) { cluster.setupPrimary({ exec: path.resolve(__dirname, genAbcScript) @@ -176,10 +175,10 @@ function invokeWorkersToGenAbc() { } for (let i = 0; i < workerNumber; ++i) { - let workerData = { - "inputs": JSON.stringify(splitedBundles[i]), - "cmd": cmdPrefix - } + const workerData = { + 'inputs': JSON.stringify(splitedBundles[i]), + 'cmd': cmdPrefix + }; cluster.fork(workerData); } @@ -189,34 +188,9 @@ function invokeWorkersToGenAbc() { process.on('exit', (code) => { if (buildPathInfo.indexOf(ARK)) { - const hashPath = genHashJsonPath(buildPathInfo); - const hashFilePath = path.join(hashPath, hashFile); - const parent: string = path.join(hashFilePath, '..'); - if (!(fs.existsSync(parent) && !fs.statSync(parent).isFile())) { - mkDir(parent); - } - for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { - let input = fileterIntermediateJsBundle[i].path; - let abcPath = input.replace(/_.js$/, '.abc'); - if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const inputContent = fs.readFileSync(input); - const hashInput = createHash('sha256'); - hashInput.update(inputContent); - const hashInputContentData = hashInput.digest('hex'); - const abcContent = fs.readFileSync(abcPath); - const hashAbc = createHash('sha256'); - hashAbc.update(abcContent); - const hashAbcContentData = hashAbc.digest('hex'); - hashJsonObject[input] = hashInputContentData; - hashJsonObject[abcPath] = hashAbcContentData; - } - if (fs.existsSync(input)) { - fs.unlinkSync(input); - } - } - fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); + writeHashJson(); } - }) + }); } } @@ -224,28 +198,28 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil for (let i = 0; i < inputPaths.length; ++i) { fileterIntermediateJsBundle.push(inputPaths[i]); } - let updateJsonObject = {}; + const hashFilePath = genHashJsonPath(buildPath); + if (hashFilePath.length === 0) { + return; + } + const updateJsonObject = {}; if (buildPath.indexOf(ARK)) { - const hashPath = genHashJsonPath(buildPath); - const hashFilePath = path.join(hashPath, hashFile); let jsonObject = {}; - let jsonFile = ""; + let jsonFile = ''; if (fs.existsSync(hashFilePath)) { jsonFile = fs.readFileSync(hashFilePath).toString(); jsonObject = JSON.parse(jsonFile); fileterIntermediateJsBundle = []; for (let i = 0; i < inputPaths.length; ++i) { - let input = inputPaths[i].path; - let abcPath = input.replace(/_.js$/, '.abc'); - if (fs.existsSync(input) && fs.existsSync(abcPath)) { - const inputContent = fs.readFileSync(input); - const hashInput = createHash('sha256'); - hashInput.update(inputContent); - const hashInputContentData = hashInput.digest('hex'); - const abcContent = fs.readFileSync(abcPath); - const hashAbc = createHash('sha256'); - hashAbc.update(abcContent); - const hashAbcContentData = hashAbc.digest('hex'); + const input = inputPaths[i].path; + const abcPath = input.replace(/_.js$/, '.abc'); + if (!fs.existsSync(input)) { + logger.error(red, `ETS:ERROR ${input} is lost`, reset); + continue; + } + if (fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { updateJsonObject[input] = hashInputContentData; updateJsonObject[abcPath] = hashAbcContentData; @@ -263,8 +237,34 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil hashJsonObject = updateJsonObject; } +function writeHashJson() { + const hashFilePath = genHashJsonPath(buildPathInfo); + if (hashFilePath.length === 0) { + return; + } + for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { + const input = fileterIntermediateJsBundle[i].path; + const abcPath = input.replace(/_.js$/, '.abc'); + if (!fs.existsSync(input) || !fs.existsSync(abcPath)) { + logger.error(red, `ETS:ERROR ${input} is lost`, reset); + continue; + } + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + hashJsonObject[input] = hashInputContentData; + hashJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); + } + fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); +} + function genHashJsonPath(buildPath: string) { buildPath = toUnixPath(buildPath); const dataTmps = buildPath.split(ARK); - return path.join(dataTmps[0], ARK); + const hashPath = path.join(dataTmps[0], ARK); + if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { + logger.error(red, `ETS:ERROR hash path does not exist`, reset); + return ''; + } + return path.join(hashPath, hashFile); } diff --git a/compiler/src/utils.ts b/compiler/src/utils.ts index d491a4148..7af185cc9 100644 --- a/compiler/src/utils.ts +++ b/compiler/src/utils.ts @@ -16,6 +16,7 @@ import ts from 'typescript'; import path from 'path'; import fs from 'fs'; +import { createHash } from 'crypto'; export enum LogType { ERROR = 'ERROR', @@ -214,4 +215,11 @@ export function toUnixPath(data: string): string { return newData; } return data; -} \ No newline at end of file +} + +export function toHashData(path: string) { + const content = fs.readFileSync(path); + const hash = createHash('sha256'); + hash.update(content); + return hash.digest('hex'); +} -- Gitee From 7ac0fc11dde2ed030ab2ffafbf4f74c6faec5a3f Mon Sep 17 00:00:00 2001 From: zhangrengao Date: Wed, 16 Mar 2022 15:04:59 +0800 Subject: [PATCH 4/4] fix hash path info Signed-off-by: zhangrengao Change-Id: Ic7ae8825f929bd21b94da5982d318516b4b59272 --- compiler/src/gen_abc_plugin.ts | 79 ++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/compiler/src/gen_abc_plugin.ts b/compiler/src/gen_abc_plugin.ts index 57628d0f3..26bc3e3a2 100644 --- a/compiler/src/gen_abc_plugin.ts +++ b/compiler/src/gen_abc_plugin.ts @@ -187,9 +187,7 @@ function invokeWorkersToGenAbc() { }); process.on('exit', (code) => { - if (buildPathInfo.indexOf(ARK)) { - writeHashJson(); - } + writeHashJson(); }); } } @@ -203,33 +201,31 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil return; } const updateJsonObject = {}; - if (buildPath.indexOf(ARK)) { - let jsonObject = {}; - let jsonFile = ''; - if (fs.existsSync(hashFilePath)) { - jsonFile = fs.readFileSync(hashFilePath).toString(); - jsonObject = JSON.parse(jsonFile); - fileterIntermediateJsBundle = []; - for (let i = 0; i < inputPaths.length; ++i) { - const input = inputPaths[i].path; - const abcPath = input.replace(/_.js$/, '.abc'); - if (!fs.existsSync(input)) { - logger.error(red, `ETS:ERROR ${input} is lost`, reset); - continue; - } - if (fs.existsSync(abcPath)) { - const hashInputContentData = toHashData(input); - const hashAbcContentData = toHashData(abcPath); - if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { - updateJsonObject[input] = hashInputContentData; - updateJsonObject[abcPath] = hashAbcContentData; - fs.unlinkSync(input); - } else { - fileterIntermediateJsBundle.push(inputPaths[i]); - } + let jsonObject = {}; + let jsonFile = ''; + if (fs.existsSync(hashFilePath)) { + jsonFile = fs.readFileSync(hashFilePath).toString(); + jsonObject = JSON.parse(jsonFile); + fileterIntermediateJsBundle = []; + for (let i = 0; i < inputPaths.length; ++i) { + const input = inputPaths[i].path; + const abcPath = input.replace(/_.js$/, '.abc'); + if (!fs.existsSync(input)) { + logger.error(red, `ETS:ERROR ${input} is lost`, reset); + continue; + } + if (fs.existsSync(abcPath)) { + const hashInputContentData = toHashData(input); + const hashAbcContentData = toHashData(abcPath); + if (jsonObject[input] === hashInputContentData && jsonObject[abcPath] === hashAbcContentData) { + updateJsonObject[input] = hashInputContentData; + updateJsonObject[abcPath] = hashAbcContentData; + fs.unlinkSync(input); } else { fileterIntermediateJsBundle.push(inputPaths[i]); } + } else { + fileterIntermediateJsBundle.push(inputPaths[i]); } } } @@ -238,10 +234,6 @@ function filterIntermediateJsBundleByHashJson(buildPath: string, inputPaths: Fil } function writeHashJson() { - const hashFilePath = genHashJsonPath(buildPathInfo); - if (hashFilePath.length === 0) { - return; - } for (let i = 0; i < fileterIntermediateJsBundle.length; ++i) { const input = fileterIntermediateJsBundle[i].path; const abcPath = input.replace(/_.js$/, '.abc'); @@ -255,16 +247,31 @@ function writeHashJson() { hashJsonObject[abcPath] = hashAbcContentData; fs.unlinkSync(input); } + const hashFilePath = genHashJsonPath(buildPathInfo); + if (hashFilePath.length === 0) { + return; + } fs.writeFileSync(hashFilePath, JSON.stringify(hashJsonObject)); } function genHashJsonPath(buildPath: string) { buildPath = toUnixPath(buildPath); - const dataTmps = buildPath.split(ARK); - const hashPath = path.join(dataTmps[0], ARK); - if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { - logger.error(red, `ETS:ERROR hash path does not exist`, reset); + if (process.env.cachePath) { + if (!fs.existsSync(process.env.cachePath) || !fs.statSync(process.env.cachePath).isDirectory()) { + logger.error(red, `ETS:ERROR hash path does not exist`, reset); + return ''; + } + return path.join(process.env.cachePath, hashFile); + } else if (buildPath.indexOf(ARK) >= 0) { + const dataTmps = buildPath.split(ARK); + const hashPath = path.join(dataTmps[0], ARK); + if (!fs.existsSync(hashPath) || !fs.statSync(hashPath).isDirectory()) { + logger.error(red, `ETS:ERROR hash path does not exist`, reset); + return ''; + } + return path.join(hashPath, hashFile); + } else { + logger.debug(red, `ETS:ERROR not cache exist`, reset); return ''; } - return path.join(hashPath, hashFile); } -- Gitee