From 9fb24444fb7b103c2940742c47486edbe0263310 Mon Sep 17 00:00:00 2001 From: zenghang Date: Sat, 21 Jun 2025 22:39:00 +0800 Subject: [PATCH] declgen thread Issue: xx Signed-off-by: zenghang Change-Id: I2f624a3bf3dea68a910b7337ca663fd573e83dcf --- .../ark_compiler/interop/declgen_worker.ts | 31 ++++++ .../interop/run_declgen_standalone.ts | 97 +++++++++++++++---- .../ark_compiler/interop/thread_pool.ts | 65 +++++++++++++ 3 files changed, 175 insertions(+), 18 deletions(-) create mode 100644 compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts create mode 100644 compiler/src/fast_build/ark_compiler/interop/thread_pool.ts diff --git a/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts b/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts new file mode 100644 index 000000000..807afde2f --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts @@ -0,0 +1,31 @@ +// declgen/declgen_worker.ts +import { parentPort } from 'worker_threads'; +import * as fs from 'fs'; +import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; +import { processInteropUI } from '../../../process_interop_ui'; +import type { RunnerParms } from './type'; + +if (!parentPort) { + throw new Error('This file should be run as a Worker'); +} + +parentPort.on('message', async (config: RunnerParms) => { + try { + if (!fs.existsSync(config.outDir)) { + fs.mkdirSync(config.outDir, { recursive: true }); + } + + generateInteropDecls(config); + + if (config.outDir) { + processInteropUI(config.outDir); + } + + parentPort?.postMessage({ status: 'done' }); + } catch (error: any) { + parentPort?.postMessage({ + status: 'error', + error: error?.message || String(error) + }); + } +}); diff --git a/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts b/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts index b1cb71d67..1b44e5a92 100644 --- a/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts +++ b/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts @@ -37,27 +37,58 @@ import { EXTNAME_D_ETS, EXTNAME_JS } from '../common/ark_define'; import { getRealModulePath } from '../../system_api/api_check_utils'; import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; import { calculateFileHash } from '../utils'; +import { runWithWorkerPool } from './thread_pool'; -export function run(param: Params): boolean { - FileManager.init(param.dependentModuleMap); - DeclfileProductor.init(param); - param.tasks.forEach(task => { - const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); - if (moduleInfo?.dynamicFiles.length <= 0) { - return; - } - if (task.buildTask === BuildType.DECLGEN) { - DeclfileProductor.getInstance().runDeclgen(moduleInfo); - } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { - DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); - } else if (task.buildTask === BuildType.BYTE_CODE_HAR) { - //todo - } - }); - FileManager.cleanFileManagerObject(); - return true; +export async function run(param: Params): Promise { + FileManager.init(param.dependentModuleMap); + DeclfileProductor.init(param); + + const declgenTasks: ArkTSEvolutionModule[] = []; + + for (const task of param.tasks) { + const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); + if (task.buildTask === BuildType.DECLGEN) { + declgenTasks.push(moduleInfo); + } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { + DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); + } + } + + await runWithWorkerPool( + declgenTasks, + 4, + (moduleInfo) => DeclfileProductor.getInstance().getWorkerPayload(moduleInfo), + (moduleInfo, msg) => { + if (msg.status === 'error') { + console.error(`Declgen failed for ${moduleInfo.packageName}:`, msg.error); + } + } + ); + + FileManager.cleanFileManagerObject(); + return true; } +// export function run(param: Params): boolean { +// FileManager.init(param.dependentModuleMap); +// DeclfileProductor.init(param); +// param.tasks.forEach(task => { +// const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); +// if (moduleInfo?.dynamicFiles.length <= 0) { +// return; +// } +// if (task.buildTask === BuildType.DECLGEN) { +// DeclfileProductor.getInstance().runDeclgen(moduleInfo); +// } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { +// DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); +// } else if (task.buildTask === BuildType.BYTE_CODE_HAR) { +// //todo +// } +// }); +// FileManager.cleanFileManagerObject(); +// return true; +// } + class DeclfileProductor { private static declFileProductor: DeclfileProductor; @@ -255,6 +286,36 @@ class DeclfileProductor { ]; DeclfileProductor.sdkConfigs = [...DeclfileProductor.defaultSdkConfigs]; } + + getWorkerPayload(moduleInfo: ArkTSEvolutionModule): any { + const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; + let existingCache = {}; + const filesToProcess: string[] = []; + const hashMap = {}; + + if (fs.existsSync(cachePath)) { + existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); + } + + moduleInfo.dynamicFiles.forEach(file => { + const unixPath = file.replace(/\\/g, '/'); + const hash = calculateFileHash(file); + if (!existingCache[unixPath] || existingCache[unixPath] !== hash) { + filesToProcess.push(unixPath); + hashMap[unixPath] = hash; + } + }); + + return { + inputDirs: [], + inputFiles: filesToProcess, + outDir: moduleInfo.declgenV2OutPath, + rootDir: moduleInfo.modulePath, + customResolveModuleNames: null, + customCompilerOptions: DeclfileProductor.compilerOptions, + includePaths: [moduleInfo.modulePath] + }; + } } function resolveModuleNames(moduleNames: string[], containingFile: string): ts.ResolvedModuleFull[] { diff --git a/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts b/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts new file mode 100644 index 000000000..44830eac5 --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts @@ -0,0 +1,65 @@ +import { Worker } from 'worker_threads'; +import path from 'path'; + +/** + * @param tasks 任务列表 + * @param maxConcurrency 最大并发线程数 + * @param getWorkerPayload 将任务对象转换为传给 Worker 的数据 + * @param onWorkerMessage 可选：处理 Worker 返回的消息 + */ +export async function runWithWorkerPool( + tasks: T[], + maxConcurrency: number, + getWorkerPayload: (task: T) => any, + onWorkerMessage?: (task: T, msg: any) => void +): Promise { + const queue = [...tasks]; + const activeWorkers: Promise[] = []; + + const runWorker = async (task: T): Promise => { + return new Promise((resolve, reject) => { + const workerPath = path.resolve(__dirname, './declgen_worker.js'); + const worker = new Worker(workerPath); + + const payload = getWorkerPayload(task); + worker.postMessage(payload); + + worker.on('message', (msg: any) => { + if (onWorkerMessage) { + onWorkerMessage(task, msg); + } + if (msg.status === 'done') { + resolve(); + } else if (msg.status === 'error') { + reject(new Error(msg.error || 'Worker encountered an error')); + } + }); + + worker.on('error', (err) => { + reject(err); + }); + + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker stopped with exit code ${code}`)); + } + }); + }); + }; + + for (let i = 0; i < maxConcurrency; i++) { + const loop = async () => { + while (queue.length > 0) { + const task = queue.shift(); + if (task) { + await runWorker(task).catch((err) => { + console.error(`[thread_pool] Worker task failed:`, err.message); + }); + } + } + }; + activeWorkers.push(loop()); + } + + await Promise.all(activeWorkers); +} -- Gitee