From 5638c54ca7a7443d31708f9e963e35ab8f7a0409 Mon Sep 17 00:00:00 2001 From: zenghang Date: Sat, 21 Jun 2025 22:39:00 +0800 Subject: [PATCH 1/2] declgen thread Issue: xx Signed-off-by: zenghang Change-Id: I2f624a3bf3dea68a910b7337ca663fd573e83dcf --- .../ark_compiler/interop/declgen_worker.ts | 31 ++++++ .../interop/run_declgen_standalone.ts | 97 +++++++++++++++---- .../ark_compiler/interop/thread_pool.ts | 65 +++++++++++++ 3 files changed, 175 insertions(+), 18 deletions(-) create mode 100644 compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts create mode 100644 compiler/src/fast_build/ark_compiler/interop/thread_pool.ts diff --git a/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts b/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts new file mode 100644 index 000000000..807afde2f --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts @@ -0,0 +1,31 @@ +// declgen/declgen_worker.ts +import { parentPort } from 'worker_threads'; +import * as fs from 'fs'; +import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; +import { processInteropUI } from '../../../process_interop_ui'; +import type { RunnerParms } from './type'; + +if (!parentPort) { + throw new Error('This file should be run as a Worker'); +} + +parentPort.on('message', async (config: RunnerParms) => { + try { + if (!fs.existsSync(config.outDir)) { + fs.mkdirSync(config.outDir, { recursive: true }); + } + + generateInteropDecls(config); + + if (config.outDir) { + processInteropUI(config.outDir); + } + + parentPort?.postMessage({ status: 'done' }); + } catch (error: any) { + parentPort?.postMessage({ + status: 'error', + error: error?.message || String(error) + }); + } +}); diff --git a/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts b/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts index b1cb71d67..1b44e5a92 100644 --- a/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts +++ b/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts @@ -37,27 +37,58 @@ import { EXTNAME_D_ETS, EXTNAME_JS } from '../common/ark_define'; import { getRealModulePath } from '../../system_api/api_check_utils'; import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; import { calculateFileHash } from '../utils'; +import { runWithWorkerPool } from './thread_pool'; -export function run(param: Params): boolean { - FileManager.init(param.dependentModuleMap); - DeclfileProductor.init(param); - param.tasks.forEach(task => { - const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); - if (moduleInfo?.dynamicFiles.length <= 0) { - return; - } - if (task.buildTask === BuildType.DECLGEN) { - DeclfileProductor.getInstance().runDeclgen(moduleInfo); - } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { - DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); - } else if (task.buildTask === BuildType.BYTE_CODE_HAR) { - //todo - } - }); - FileManager.cleanFileManagerObject(); - return true; +export async function run(param: Params): Promise { + FileManager.init(param.dependentModuleMap); + DeclfileProductor.init(param); + + const declgenTasks: ArkTSEvolutionModule[] = []; + + for (const task of param.tasks) { + const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); + if (task.buildTask === BuildType.DECLGEN) { + declgenTasks.push(moduleInfo); + } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { + DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); + } + } + + await runWithWorkerPool( + declgenTasks, + 4, + (moduleInfo) => DeclfileProductor.getInstance().getWorkerPayload(moduleInfo), + (moduleInfo, msg) => { + if (msg.status === 'error') { + console.error(`Declgen failed for ${moduleInfo.packageName}:`, msg.error); + } + } + ); + + FileManager.cleanFileManagerObject(); + return true; } +// export function run(param: Params): boolean { +// FileManager.init(param.dependentModuleMap); +// DeclfileProductor.init(param); +// param.tasks.forEach(task => { +// const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); +// if (moduleInfo?.dynamicFiles.length <= 0) { +// return; +// } +// if (task.buildTask === BuildType.DECLGEN) { +// DeclfileProductor.getInstance().runDeclgen(moduleInfo); +// } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { +// DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); +// } else if (task.buildTask === BuildType.BYTE_CODE_HAR) { +// //todo +// } +// }); +// FileManager.cleanFileManagerObject(); +// return true; +// } + class DeclfileProductor { private static declFileProductor: DeclfileProductor; @@ -255,6 +286,36 @@ class DeclfileProductor { ]; DeclfileProductor.sdkConfigs = [...DeclfileProductor.defaultSdkConfigs]; } + + getWorkerPayload(moduleInfo: ArkTSEvolutionModule): any { + const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; + let existingCache = {}; + const filesToProcess: string[] = []; + const hashMap = {}; + + if (fs.existsSync(cachePath)) { + existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); + } + + moduleInfo.dynamicFiles.forEach(file => { + const unixPath = file.replace(/\\/g, '/'); + const hash = calculateFileHash(file); + if (!existingCache[unixPath] || existingCache[unixPath] !== hash) { + filesToProcess.push(unixPath); + hashMap[unixPath] = hash; + } + }); + + return { + inputDirs: [], + inputFiles: filesToProcess, + outDir: moduleInfo.declgenV2OutPath, + rootDir: moduleInfo.modulePath, + customResolveModuleNames: null, + customCompilerOptions: DeclfileProductor.compilerOptions, + includePaths: [moduleInfo.modulePath] + }; + } } function resolveModuleNames(moduleNames: string[], containingFile: string): ts.ResolvedModuleFull[] { diff --git a/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts b/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts new file mode 100644 index 000000000..44830eac5 --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts @@ -0,0 +1,65 @@ +import { Worker } from 'worker_threads'; +import path from 'path'; + +/** + * @param tasks 任务列表 + * @param maxConcurrency 最大并发线程数 + * @param getWorkerPayload 将任务对象转换为传给 Worker 的数据 + * @param onWorkerMessage 可选：处理 Worker 返回的消息 + */ +export async function runWithWorkerPool( + tasks: T[], + maxConcurrency: number, + getWorkerPayload: (task: T) => any, + onWorkerMessage?: (task: T, msg: any) => void +): Promise { + const queue = [...tasks]; + const activeWorkers: Promise[] = []; + + const runWorker = async (task: T): Promise => { + return new Promise((resolve, reject) => { + const workerPath = path.resolve(__dirname, './declgen_worker.js'); + const worker = new Worker(workerPath); + + const payload = getWorkerPayload(task); + worker.postMessage(payload); + + worker.on('message', (msg: any) => { + if (onWorkerMessage) { + onWorkerMessage(task, msg); + } + if (msg.status === 'done') { + resolve(); + } else if (msg.status === 'error') { + reject(new Error(msg.error || 'Worker encountered an error')); + } + }); + + worker.on('error', (err) => { + reject(err); + }); + + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker stopped with exit code ${code}`)); + } + }); + }); + }; + + for (let i = 0; i < maxConcurrency; i++) { + const loop = async () => { + while (queue.length > 0) { + const task = queue.shift(); + if (task) { + await runWorker(task).catch((err) => { + console.error(`[thread_pool] Worker task failed:`, err.message); + }); + } + } + }; + activeWorkers.push(loop()); + } + + await Promise.all(activeWorkers); +} -- Gitee From 4670d2ee0757ca7792349f34c796725c8d675008 Mon Sep 17 00:00:00 2001 From: luobohua <12315995+luobohua2025@user.noreply.gitee.com> Date: Thu, 3 Jul 2025 10:11:16 +0800 Subject: [PATCH 2/2] Description:multiple threads test Team:Building Feature or Bugfix:No Binary Source:No PrivateCode(Yes/No):No --- .../ark_compiler/interop/2_thread_pool.ts | 129 +++ .../ark_compiler/interop/declgen_worker.ts | 202 +++- .../ark_compiler/interop/interop_manager.ts | 15 +- .../fast_build/ark_compiler/interop/main.ts | 302 ++++++ .../ark_compiler/interop/record_time_mem.ts | 202 ++++ .../interop/run_declgen_standalone.ts | 977 +++++++++++------- .../ark_compiler/interop/test_all.ts | 106 ++ .../ark_compiler/interop/thread_pool.ts | 175 +++- .../fast_build/ark_compiler/interop/type.ts | 15 + 9 files changed, 1683 insertions(+), 440 deletions(-) create mode 100644 compiler/src/fast_build/ark_compiler/interop/2_thread_pool.ts create mode 100644 compiler/src/fast_build/ark_compiler/interop/main.ts create mode 100644 compiler/src/fast_build/ark_compiler/interop/record_time_mem.ts create mode 100644 compiler/src/fast_build/ark_compiler/interop/test_all.ts diff --git a/compiler/src/fast_build/ark_compiler/interop/2_thread_pool.ts b/compiler/src/fast_build/ark_compiler/interop/2_thread_pool.ts new file mode 100644 index 000000000..de66f068a --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/2_thread_pool.ts @@ -0,0 +1,129 @@ +import { + ArkTSEvolutionModule, + DeclfileSideProductor +} from './type'; +// import ts from 'typescript'; +import { Worker } from 'worker_threads'; +import os from 'os'; +import path from 'path'; + +// 线程池管理类 +class ThreadPool { + private maxThreads: number; + private taskQueue: Array<{ task: () => Promise; resolve: () => void; reject: (error: Error) => void }>; + private activeCount: number; + + constructor(maxThreads = Math.max(1, Math.min(os.cpus().length - 1, 4))) { + this.maxThreads = maxThreads; + this.taskQueue = []; + this.activeCount = 0; + } + + run(task: () => Promise): Promise { + return new Promise((resolve, reject) => { + this.taskQueue.push({ task, resolve, reject }); + this.processQueue(); + }); + } + + private processQueue() { + while (this.activeCount < this.maxThreads && this.taskQueue.length > 0) { + const { task, resolve, reject } = this.taskQueue.shift()!; + this.activeCount++; + + task() + .then(resolve) + .catch(reject) + .finally(() => { + this.activeCount--; + this.processQueue(); + }); + } + } +} + +// 任务分发器 +export class TaskDispatcher { + private static instance: TaskDispatcher; + private threadPool: ThreadPool; + + private constructor() { + this.threadPool = new ThreadPool(); + } + + static getInstance(): TaskDispatcher { + if (!this.instance) { + this.instance = new TaskDispatcher(); + } + return this.instance; + } + + async runInThread(moduleInfo: ArkTSEvolutionModule, + sideProductor: DeclfileSideProductor): Promise { + return this.threadPool.run(() => { + return new Promise((resolve, reject) => { + const worker = new Worker(path.join(__dirname, './declgen_worker.js'), { + workerData: { + moduleInfo, + sideProductor + }}); + + console.log('working'); + + worker.on('message', (message) => { + if (message.success) { + worker.terminate(); // 终止线程 + resolve(); + console.log('done and terminate'); + } else if (message.error) { + reject(new Error(message.error)); + console.log('Worker encountered an error', message.error); + } + }); + worker.on('error', reject); + worker.on('exit', (code) => { + if (code !== 0) { + reject(new Error(`Worker stopped with exit code ${code}`)); + } + }); + }); + }); + } +} + +// 主线程执行逻辑 +// export async function run(param: Params): Promise { +// FileManager.init(param.dependentModuleMap); +// DeclfileProductor.init(param); + +// const dispatcher = TaskDispatcher.getInstance(); +// const tasks: Promise[] = []; + +// for (const task of param.tasks) { +// const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); +// if (!moduleInfo || moduleInfo.dynamicFiles.length <= 0) { +// continue; +// } +// if (task.buildTask === BuildType.DECLGEN) { +// tasks.push( +// dispatcher.runInThread( +// moduleInfo, +// param.projectConfig as ProjectConfig, +// DeclfileProductor.compilerOptions +// ) +// ); +// } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { +// DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); +// } +// } + +// try { +// await Promise.all(tasks); +// return true; +// } catch (error) { +// console.error('Error in parallel task execution:', error); +// return false; +// } finally { +// FileManager.cleanFileManagerObject(); +// } +// } diff --git a/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts b/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts index 807afde2f..d18d8a4ee 100644 --- a/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts +++ b/compiler/src/fast_build/ark_compiler/interop/declgen_worker.ts @@ -1,31 +1,201 @@ // declgen/declgen_worker.ts -import { parentPort } from 'worker_threads'; +// import { FileManager } from './interop_manager'; +// import { parentPort } from 'worker_threads'; import * as fs from 'fs'; import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; import { processInteropUI } from '../../../process_interop_ui'; -import type { RunnerParms } from './type'; +// import type { RunnerParms } from './type'; +import { readDeaclareFiles } from '../../../ets_checker'; +import { + toUnixPath +} from '../../../utils'; +// import { Payload } from './type'; +// import { resolveModuleNames } from './run_declgen_standalone'; -if (!parentPort) { - throw new Error('This file should be run as a Worker'); -} +import { parentPort, workerData } from 'worker_threads'; +import { calculateFileHash } from '../utils'; +import { + ArkTSEvolutionModule, + DeclfileSideProductor, + DECLGEN_CACHE_FILE, + RunnerParms +} from './type'; +// import ts from 'typescript'; +import { resolveModuleNamesByThread } from './run_declgen_standalone'; +// import { FileManager } from './interop_manager'; + +// 工作线程实际执行函数 (线程安全版本) +function runDeclgenThreadSafe( + moduleInfo: ArkTSEvolutionModule, + sideProductor: DeclfileSideProductor +): void { + const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; + const filesToProcess = []; + + let existingCache = {}; + let hashMap = {}; + ({ hashMap, existingCache } = preprocess(cachePath, existingCache, moduleInfo, filesToProcess)); + if (filesToProcess.length === 0) { + return; + } + + readDeaclareFiles().forEach(filePath => { + filesToProcess.push(toUnixPath(filePath)); + }); + + const customResolveModuleNames = (moduleNames: string[], containingFile: string) => + resolveModuleNamesByThread(moduleNames, containingFile, sideProductor); + const config: RunnerParms = { + inputDirs: [], + inputFiles: filesToProcess, + outDir: moduleInfo.declgenV2OutPath, + rootDir: moduleInfo.modulePath, + customResolveModuleNames, + customCompilerOptions: sideProductor.compilerOptions, + includePaths: [moduleInfo.modulePath] + }; -parentPort.on('message', async (config: RunnerParms) => { try { if (!fs.existsSync(config.outDir)) { fs.mkdirSync(config.outDir, { recursive: true }); } generateInteropDecls(config); + processInteropUI(moduleInfo.declgenV2OutPath); - if (config.outDir) { - processInteropUI(config.outDir); - } + const newCache = { ...existingCache, ...hashMap }; + fs.writeFileSync(cachePath, JSON.stringify(newCache, null, 2)); + } catch (error) { + console.error(`Error in declgen for ${moduleInfo.packageName}:`, error); + throw error; + } +} - parentPort?.postMessage({ status: 'done' }); - } catch (error: any) { - parentPort?.postMessage({ - status: 'error', - error: error?.message || String(error) - }); +function preprocess(cachePath: string, existingCache: {}, moduleInfo: ArkTSEvolutionModule, filesToProcess: string[]) { + if (fs.existsSync(cachePath)) { + try { + existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); + } catch (e) { + console.warn(`Failed to parse cache file ${cachePath}: ${e}`); + } } -}); + + const hashMap = {}; + moduleInfo.dynamicFiles.forEach(filePath => { + const unixPath = toUnixPath(filePath); + try { + const fileHash = calculateFileHash(filePath); + if (!existingCache[unixPath] || existingCache[unixPath] !== fileHash) { + filesToProcess.push(unixPath); + hashMap[unixPath] = fileHash; + } + } catch (error) { + console.error(`Error processing file ${filePath}:`, error); + } + }); + return { hashMap, existingCache }; +} + +try { + console.log('working'); + const { moduleInfo, sideProductor } = workerData; + runDeclgenThreadSafe(moduleInfo, sideProductor); + parentPort.postMessage({ success: true }); +} catch (error) { + parentPort.postMessage({ + error: error.message || 'Unknown error in worker thread' + }); +} + +// // 工作线程执行逻辑 +// console.log(parentPort, isMainThread, workerData.__isWorker); +// if (!isMainThread && workerData.__isWorker) { +// const { moduleInfo, projectConfig, compilerOptions } = workerData; + +// // 复制声明文件生成逻辑（线程安全版本） +// const runDeclgenThreadSafe = ( +// moduleInfo: ArkTSEvolutionModule, +// projectConfig: ProjectConfig, +// compilerOptions: ts.CompilerOptions +// ): void => { +// const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; +// let existingCache = {}; +// const filesToProcess: string[] = []; + +// let hashMap: Record; +// ({ hashMap, existingCache } = preprocess(cachePath, existingCache, moduleInfo, filesToProcess)); + +// if (filesToProcess.length === 0) { +// return; +// } + +// readDeaclareFiles().forEach(filePath => { +// filesToProcess.push(toUnixPath(filePath)); +// }); + +// const config: RunnerParms = { +// inputDirs: [], +// inputFiles: filesToProcess, +// outDir: moduleInfo.declgenV2OutPath, +// rootDir: moduleInfo.modulePath, +// customResolveModuleNames: resolveModuleNames, +// customCompilerOptions: compilerOptions, +// includePaths: [moduleInfo.modulePath] +// }; + +// try { +// if (!fs.existsSync(config.outDir)) { +// fs.mkdirSync(config.outDir, { recursive: true }); +// } + +// generateInteropDecls(config); +// processInteropUI(moduleInfo.declgenV2OutPath); + +// const newCache = { ...existingCache, ...hashMap }; +// fs.writeFileSync(cachePath, JSON.stringify(newCache, null, 2)); +// } catch (error) { +// console.error(`Error in declgen for ${moduleInfo.packageName}:`, error); +// throw error; +// } +// }; + +// try { +// runDeclgenThreadSafe(moduleInfo, projectConfig, compilerOptions); +// console.log('success'); +// parentPort?.postMessage({ success: true }); +// } catch (error) { +// parentPort?.postMessage({ error }); +// } +// } + +// if (!parentPort) { +// throw new Error('This file should be run as a Worker'); +// } + +// parentPort.on('message', async(payload: Payload) => { +// try { +// if (payload.config.inputFiles.length === 0) { +// parentPort?.postMessage({ status: 'done' }); +// return; +// } +// // readDeaclareFiles().forEach(path => { +// // payload.config.inputFiles.push(toUnixPath(path)); +// // }); + +// if (!fs.existsSync(payload.config.outDir)) { +// fs.mkdirSync(payload.config.outDir, { recursive: true }); +// } + +// // payload.config.customResolveModuleNames = resolveModuleNames; +// generateInteropDecls(payload.config); +// processInteropUI(FileManager.arkTSModuleMap.get(payload.packageName)?.declgenV2OutPath); +// fs.writeFileSync(payload.cachePath, JSON.stringify(payload.newCache, null, 2)); + +// parentPort?.postMessage({ status: 'done' }); +// } catch (error: any) { +// parentPort?.postMessage({ +// status: 'error', +// error: error?.message || String(error) +// }); +// } +// }); diff --git a/compiler/src/fast_build/ark_compiler/interop/interop_manager.ts b/compiler/src/fast_build/ark_compiler/interop/interop_manager.ts index d099314ad..0cd1b2d38 100644 --- a/compiler/src/fast_build/ark_compiler/interop/interop_manager.ts +++ b/compiler/src/fast_build/ark_compiler/interop/interop_manager.ts @@ -47,7 +47,7 @@ import { ARKTS_HYBRID } from './pre_define'; -export let entryFileLanguageInfo = new Map(); +export const entryFileLanguageInfo = new Map(); export class FileManager { private static instance: FileManager | undefined = undefined; @@ -62,8 +62,6 @@ export class FileManager { static isInteropSDKEnabled: boolean = false; static sharedObj: Object | undefined = undefined; - private constructor() { } - public static init( dependentModuleMap: Map, aliasPaths?: Map, @@ -123,7 +121,7 @@ export class FileManager { declgenV1OutPath: module.declgenV1OutPath ? toUnixPath(module.declgenV1OutPath) : undefined, declgenV2OutPath: module.declgenV2OutPath ? toUnixPath(module.declgenV2OutPath) : undefined, declgenBridgeCodePath: module.declgenBridgeCodePath ? toUnixPath(module.declgenBridgeCodePath) : undefined, - declFilesPath: module.declFilesPath ? toUnixPath(module.declFilesPath) : undefined, + declFilesPath: module.declFilesPath ? toUnixPath(module.declFilesPath) : undefined }; convertedMap.set(key, convertedModule); } @@ -316,7 +314,7 @@ export class FileManager { const sdkMatches: [Set | undefined, string][] = [ [FileManager.dynamicLibPath, ARKTS_1_1], [FileManager.staticSDKDeclPath, ARKTS_1_2], - [FileManager.staticSDKGlueCodePath, ARKTS_1_2], + [FileManager.staticSDKGlueCodePath, ARKTS_1_2] ]; for (const [paths, version] of sdkMatches) { @@ -398,13 +396,13 @@ export function collectSDKInfo(share: Object): { const staticSDKInteropDecl: Set = new Set([ path.resolve(staticInteroSDKBasePath, './declarations/kits'), path.resolve(staticInteroSDKBasePath, './declarations/api'), - path.resolve(staticInteroSDKBasePath, './declarations/arkts'), + path.resolve(staticInteroSDKBasePath, './declarations/arkts') ].map(toUnixPath)); const staticSDKGlueCodePath: Set = new Set([ path.resolve(staticInteroSDKBasePath, './bridge/kits'), path.resolve(staticInteroSDKBasePath, './bridge/api'), - path.resolve(staticInteroSDKBasePath, './bridge/arkts'), + path.resolve(staticInteroSDKBasePath, './bridge/arkts') ].map(toUnixPath)); const declarationsPath: string = path.resolve(share.projectConfig.etsLoaderPath, './declarations').replace(/\\/g, '/'); @@ -491,7 +489,6 @@ export function processAbilityPagesFullPath(abilityPagesFullPath: Set): } } - export function transformAbilityPages(abilityPath: string): boolean { const entryBridgeCodePath = process.env.entryBridgeCodePath; if (!entryBridgeCodePath) { @@ -517,7 +514,7 @@ export function transformAbilityPages(abilityPath: string): boolean { } function transformModuleNameToRelativePath(moduleName): string { - let defaultSourceRoot = 'src/main'; + const defaultSourceRoot = 'src/main'; const normalizedModuleName = moduleName.replace(/\\/g, '/'); const normalizedRoot = defaultSourceRoot.replace(/\\/g, '/'); diff --git a/compiler/src/fast_build/ark_compiler/interop/main.ts b/compiler/src/fast_build/ark_compiler/interop/main.ts new file mode 100644 index 000000000..684ea3173 --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/main.ts @@ -0,0 +1,302 @@ +// let param = { +// tasks: [ +// { +// packageName: "application", +// buildModule: "declgen" +// } +// ], +// dependentModuleMap: { +// 'application': { +// language: '1.1', +// packageName: 'application', +// moduleName: 'entry', +// modulePath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application', +// declgenV1OutPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application\\build\\default\\intermediates\\declgen\\default\\declgenV1', +// declgenV2OutPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application\\build\\default\\intermediates\\declgen\\default\\declgenV2', +// eslint-disable-next-line max-len +// declgenBridgeCodePath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', +// declFilesPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\application\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', +// fileList: [ +// "D:/temp/helloworld_0331/helloworld_0331/application/src/main/ets/pages/Index.ets" +// ], +// cachePath:"D:\\outDemo", +// byteCodeHarInfo: { +// bchar: { +// abcPath: "D:\\modules.abc", +// } +// }, +// pkgPath:'D:\\temp\\helloworld_0331\\helloworld_0331\\application', +// packageVersion:'1.0.0' +// }, +// 'library': { +// language: '1.2', +// packageName: 'library', +// moduleName: 'library', +// pkgPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library', +// modulePath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library', +// pkgPath:'D:\\temp\\helloworld_0331\\helloworld_0331\\library', +// declgenV1OutPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library\\build\\default\\intermediates\\declgen\\default\\declgenV1', +// declgenV2OutPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library\\build\\default\\intermediates\\declgen\\default\\declgenV2', +// declgenBridgeCodePath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', +// declFilesPath: 'D:\\temp\\helloworld_0331\\helloworld_0331\\library\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', +// fileList: [ +// "D:/temp/helloworld_0331/helloworld_0331/application/src/main/ets/pages/Index.ets" +// ] +// } +// }, +// projectConfig:{ +// bundleName: "application", +// mainModuleName: "mainModuleName", +// projectRootPath: "D:/temp/helloworld_0331/helloworld_0331", +// } +// } +// let param = { +// tasks: [ +// {packageName: 'library', buildTask: 'declgen'}, +// {packageName: 'mixhar', buildTask: 'declgen'} +// ], +// dependentModuleMap: new Map([ +// ['library', { +// language: '1.1', +// packageName: 'library', +// moduleName: 'library', +// modulePath: 'D:\\\\projectTest\\\\MyApplication38\\\\library', +// declgenV2OutPath: 'D:\\\\projectTest\\\\MyApplication38\\\\library\\\\build\\\\default\\\\intermediates\\\\declgen\\\\default\\\\declgenV2', +// dynamicFiles:[ +// 'D:\\projectTest\\MyApplication38\\library\\Index.ets', +// 'D:\\projectTest\\MyApplication38\\library\\src\\main\\ets\\components\\cccc.ets', +// 'D:\\projectTest\\MyApplication38\\library\\src\\main\\ets\\components\\MainPage.ets' +// ], +// cachePath:'D:\\projectTest\\MyApplication38\\library\\build\\default\\intermediates\\declgen\\default\\cache', +// pkgPath: 'D:\\projectTest\\MyApplication38\\library', +// packageVersion: '1.0.0' +// }], +// ['mixhar', { +// language: 'hybrid', +// packageName: 'mixhar', +// moduleName: 'mixHar', +// modulePath: 'D:\\projectTest\\MyApplication38\\mixHar', +// declFilesPath: 'D:\\projectTest\\MyApplication38\\mixHar\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', +// declgenV1OutPath: 'D:\\projectTest\\MyApplication38\\mixHar\\build\\default\\intermediates\\declgen\\default\\declgenV1', +// declgenV2OutPath: 'D:\\projectTest\\MyApplication38\\mixHar\\build\\default\\intermediates\\declgen\\default\\declgenV2', +// declgenBridgeCodePath: 'D:\\projectTest\\MyApplication38\\mixHar\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', +// dynamicFiles: [ +// 'D:\\projectTest\\MyApplication38\\mixHar\\Index.ets' +// ], +// cachePath: 'D:\\projectTest\\MyApplication38\\build\\declgen\\mixhar\\cache', +// pkgPath: 'D:\\projectTest\\MyApplication38\\mixHar', +// packageVersion: '1.0.0' +// }] +// ]) +// , +// projectConfig: { +// bundleName: 'com.example.myapplication', +// projectRootPath: 'D:\\projectTest\\MyApplication38' +// } +// } + +const declgen = require('D:\\SDK\\OH\\20\\20\\ets\\ets1.1\\build-tools\\ets-loader\\lib\\fast_build\\ark_compiler\\interop\\run_declgen_standalone.js'); + +const param = { + tasks: [ + { + packageName: 'entry', + buildTask: 'declgen' + }, + { + packageName: '@alipay/library1', + buildTask: 'declgen' + }, + { + packageName: '@alipay/library2', + buildTask: 'declgen' + }, + { + packageName: 'entry', + buildTask: 'declgen' + }, + { + packageName: '@alipay/library1', + buildTask: 'declgen' + }, + { + packageName: '@alipay/library2', + buildTask: 'declgen' + } + ], + dependentModuleMap: new Map([ + ['entry', { + language: '1.1', + packageName: 'entry', + moduleName: 'entry', + modulePath: 'D:\\code\\openharmony\\mytest\\my-test\\entry', + declFilesPath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', + declgenV1OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenV1', + declgenV2OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenV2', + declgenBridgeCodePath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', + dynamicFiles: [ + 'D:\\code\\openharmony\\mytest\\my-test\\entry\\src\\main\\ets\\entryability\\EntryAbility.ets', + 'D:\\code\\openharmony\\mytest\\my-test\\entry\\src\\main\\ets\\pages\\Index.ets' + ], + staticFiles: [], + cachePath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\cache', + byteCodeHarInfo: { + 'entry': { + 'abcPath': 'D:\\code\\openharmony\\mytest\\my-test\\entry\\ets\\modules.abc' + } + }, + pkgPath: 'D:\\code\\openharmony\\mytest\\my-test\\entry', + packageVersion: '1.0.0' + }], + ['@alipay/library1', { + language: '1.1', + packageName: '@alipay/library1', + moduleName: 'library1', + modulePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1', + declFilesPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', + declgenV1OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenV1', + declgenV2OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenV2', + declgenBridgeCodePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', + dynamicFiles: [ + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\Index.ets', + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\src\\main\\ets\\components\\MainPage.ets' + ], + staticFiles: [], + cachePath: 'D:\\code\\openharmony\\mytest\\my-test\\build\\declgen\\@alipay\\library1\\cache', + byteCodeHarInfo: { + '@alipay/library1': { + 'abcPath': 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\ets\\modules.abc' + } + }, + pkgPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1', + packageVersion: '1.0.0' + }], + ['@alipay/library2', { + language: '1.1', + packageName: '@alipay/library2', + moduleName: 'library2', + modulePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2', + declFilesPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', + declgenV1OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenV1', + declgenV2OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenV2', + declgenBridgeCodePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', + dynamicFiles: [ + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\Index.ets', + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\src\\main\\ets\\components\\MainPage.ets' + ], + staticFiles: [], + cachePath: 'D:\\code\\openharmony\\mytest\\my-test\\build\\declgen\\@alipay\\library2\\cache', + byteCodeHarInfo: { + '@alipay/library2': { + 'abcPath': 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\ets\\modules.abc' + } + }, + pkgPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2', + packageVersion: '1.0.0' + }], + ['@alipay/librarystatic1', { + language: '1.2', + packageName: '@alipay/librarystatic1', + moduleName: 'libraryStatic1', + modulePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1', + declFilesPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', + declgenV1OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\build\\default\\intermediates\\declgen\\default\\declgenV1', + declgenV2OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\build\\default\\intermediates\\declgen\\default\\declgenV2', + declgenBridgeCodePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', + dynamicFiles: [], + staticFiles: [ + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\Index.ets', + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\src\\main\\ets\\components\\MainPage.ets' + ], + cachePath: 'D:\\code\\openharmony\\mytest\\my-test\\build\\declgen\\@alipay\\librarystatic1\\cache', + byteCodeHarInfo: { + '@alipay/librarystatic1': { + 'abcPath': 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1\\ets\\modules.abc' + } + }, + pkgPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\libraryStatic1', + packageVersion: '1.0.0' + }], + ['entry', { + language: '1.1', + packageName: 'entry', + moduleName: 'entry', + modulePath: 'D:\\code\\openharmony\\mytest\\my-test\\entry', + declFilesPath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', + declgenV1OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenV1', + declgenV2OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenV2', + declgenBridgeCodePath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', + dynamicFiles: [ + 'D:\\code\\openharmony\\mytest\\my-test\\entry\\src\\main\\ets\\entryability\\EntryAbility.ets', + 'D:\\code\\openharmony\\mytest\\my-test\\entry\\src\\main\\ets\\pages\\Index.ets' + ], + staticFiles: [], + cachePath: 'D:\\code\\openharmony\\mytest\\my-test\\entry\\build\\default\\intermediates\\declgen\\default\\cache', + byteCodeHarInfo: { + 'entry': { + 'abcPath': 'D:\\code\\openharmony\\mytest\\my-test\\entry\\ets\\modules.abc' + } + }, + pkgPath: 'D:\\code\\openharmony\\mytest\\my-test\\entry', + packageVersion: '1.0.0' + }], + ['@alipay/library1', { + language: '1.1', + packageName: '@alipay/library1', + moduleName: 'library1', + modulePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1', + declFilesPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', + declgenV1OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenV1', + declgenV2OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenV2', + declgenBridgeCodePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', + dynamicFiles: [ + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\Index.ets', + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\src\\main\\ets\\components\\MainPage.ets' + ], + staticFiles: [], + cachePath: 'D:\\code\\openharmony\\mytest\\my-test\\build\\declgen\\@alipay\\library1\\cache', + byteCodeHarInfo: { + '@alipay/library1': { + 'abcPath': 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1\\ets\\modules.abc' + } + }, + pkgPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library1', + packageVersion: '1.0.0' + }], + ['@alipay/library2', { + language: '1.1', + packageName: '@alipay/library2', + moduleName: 'library2', + modulePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2', + declFilesPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\decl-fileInfo.json', + declgenV1OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenV1', + declgenV2OutPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenV2', + declgenBridgeCodePath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\build\\default\\intermediates\\declgen\\default\\declgenBridgeCode', + dynamicFiles: [ + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\Index.ets', + 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\src\\main\\ets\\components\\MainPage.ets' + ], + staticFiles: [], + cachePath: 'D:\\code\\openharmony\\mytest\\my-test\\build\\declgen\\@alipay\\library2\\cache', + byteCodeHarInfo: { + '@alipay/library2': { + 'abcPath': 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2\\ets\\modules.abc' + } + }, + pkgPath: 'D:\\code\\openharmony\\mytest\\my-test\\deps\\library2', + packageVersion: '1.0.0' + }] + ]), + projectConfig: { + 'bundleName': 'com.example.alipaytest', + 'projectRootPath': 'D:\\code\\openharmony\\mytest\\my-test' + } +}; + +async function runDeclgen() { + await declgen.run(param); + + console.log('wait until finish'); +} + +runDeclgen().catch(err => console.error('Error:', err)); diff --git a/compiler/src/fast_build/ark_compiler/interop/record_time_mem.ts b/compiler/src/fast_build/ark_compiler/interop/record_time_mem.ts new file mode 100644 index 000000000..fb8df05c7 --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/record_time_mem.ts @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as fs from 'fs'; +import path from 'path'; +import { RECORD_TYPE } from './type' + +export class TimeMemData { + public startTime: number = 0; + public endtime: number = 0; + public time: number = 0; + public startMem: number = 0; + public endMem:number = 0; + public mem: number = 0; +} + +export class SingleData { + public time: number = 0; + public mem: number = 0; +} + +export class CompileSingleData { + private timeMemMap: Map; + private totalTime: number = 0; + private startTime: number = 0; + private startMem: number = 0; + private file: string = ''; + private recordType: RECORD_TYPE; + + constructor(file: string, recordType: RECORD_TYPE) { + this.file = file; + this.timeMemMap = new Map(); + this.recordType = recordType; + } + + public record(startKey: string, lastEndKey: string = '') { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + let currentTime = new Date().getTime(); + let currentMem = process.memoryUsage.rss(); + let tmp: SingleData | undefined = this.timeMemMap.get(lastEndKey); + if (tmp) { + tmp.time = currentTime - this.startTime; + tmp.mem = (currentMem > this.startMem) ? (currentMem- this.startMem) : 0; ; + this.timeMemMap.set(lastEndKey, tmp); + } + + if (startKey == '') { + return; + } + + let tmp1: SingleData | undefined = this.timeMemMap.get(startKey); + if (tmp1 == undefined) { + this.startTime = currentTime; + this.startMem = currentMem; + let data: SingleData = new SingleData(); + data.time = 0; + data.mem = 0; + this.timeMemMap.set(startKey, data); + } + } + + writeSumSingle(cachePath: string, deputyName: string = '') { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + const csvData: string[] = [ + "timeKey, time(ms), mem(M)" + ]; + this.timeMemMap.forEach((v: SingleData, k: string) => { + let element = `${k}` +', ' + `${v.time}` + 'ms' + ', ' + `${Math.round(v.mem / 1024 / 1024)}` + 'M' ; + csvData.push(element); + }); + let name = path.basename(this.file) + let currentExt = path.extname(name) + let fileWithoutExt = name.substring(0, name.lastIndexOf(currentExt)); + let fileName = `${fileWithoutExt}`+ deputyName +'_time_mem.csv'; + let filePath = path.join(cachePath, fileName); + csvData.forEach(row => { + fs.appendFileSync(filePath, `${row}\n`); + }); + } +} + + +export class RecordTimeMem { + private static instance: RecordTimeMem | undefined; + private compileType: string = ''; + private memLow: number = 0; + private memHigh: number = 0; + private timeMemMap: Map; + private recordType: RECORD_TYPE = RECORD_TYPE.DEFAULT_TYPE;; + + public static getInstance(): RecordTimeMem { + if (!RecordTimeMem.instance) { + RecordTimeMem.instance = new RecordTimeMem(); + } + return RecordTimeMem.instance; + } + + private constructor() { + this.memLow = process.memoryUsage.rss(); + this.timeMemMap = new Map(); + } + + public setCompileType(type: string) { + this.compileType = type; + } + + public initRecordType(recordType: RECORD_TYPE) { + this.recordType = recordType; + } + + public start(timeKey: string) { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + let currentTime = new Date().getTime(); + let currentMem = process.memoryUsage.rss(); + let tmp: TimeMemData | undefined = this.timeMemMap.get(timeKey); + if (tmp) { + tmp.startTime = currentTime; + tmp.startMem = currentMem; + this.timeMemMap.set(timeKey, tmp); + } else { + let data: TimeMemData = new TimeMemData(); + data.startTime = currentTime; + data.endMem = currentMem; + this.timeMemMap.set(timeKey, data); + } + this.recordMem(); + } + + public end(timeKey: string) { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + let currentTime = new Date().getTime(); + let currentMem = process.memoryUsage.rss(); + let tmp: TimeMemData | undefined = this.timeMemMap.get(timeKey); + if (tmp) { + tmp.endtime = currentTime; + tmp.endMem = currentMem; + tmp.time = tmp.endtime - tmp.startTime; + tmp.mem = (tmp.endMem > tmp.startMem) ? (tmp.endMem - tmp.startMem) : 0; + this.timeMemMap.set(timeKey, tmp); + } + this.recordMem(); + } + + + private recordMem() { + let rss: number = process.memoryUsage.rss(); + if (rss < this.memLow) { + this.memLow = rss; + } + if (rss > this.memHigh) { + this.memHigh = rss; + } + } + + writeDataCsv(filePath: string, data: any[]) { + const content = data.map(row => { + return Object.values(row).join(','); + }).join('\n'); + fs.writeFile(filePath, content, (err) => { + if (err) { + console.error('Error writing file:', err); + } + }); + } + + writeSum(cachePath: string) { + if (this.recordType == RECORD_TYPE.DEFAULT_TYPE) { + return; + } + const csvData: any[] = []; + csvData.push({ timeKey: `memory low`, time: 'undefined', mem: `${Math.round(this.memLow / 1024 / 1024)}` + 'M' }); + csvData.push({ timeKey: `memory high`, time: 'undefined', mem: `${Math.round(this.memHigh / 1024 / 1024)}` + 'M' }); + csvData.push({ timeKey: `timeKey`, time: `time` + '(ms)', mem: `mem` + '(M)' }); + this.timeMemMap.forEach((v: TimeMemData, k: string) => { + let element = { timeKey: `${k}`, time: `${v.time}` + 'ms', mem: `${Math.round(v.mem / 1024 / 1024)}` + 'M' }; + csvData.push(element); + }) + let fileName = `${this.compileType}`+'_time_mem.csv'; + let filePath = path.join(cachePath, fileName); + this.writeDataCsv(filePath, csvData); + } +} \ No newline at end of file diff --git a/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts b/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts index 1b44e5a92..17f7436ea 100644 --- a/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts +++ b/compiler/src/fast_build/ark_compiler/interop/run_declgen_standalone.ts @@ -17,486 +17,659 @@ import { FileManager } from './interop_manager'; import { ResolveModuleInfo, getResolveModule, readDeaclareFiles } from '../../../ets_checker'; import { processInteropUI } from '../../../process_interop_ui'; import { - mkdirsSync, - readFile, - toUnixPath + mkdirsSync, + readFile, + toUnixPath } from '../../../utils'; -import { - ArkTSEvolutionModule, - BuildType, - DeclFilesConfig, - DECLGEN_CACHE_FILE, - Params, - ProjectConfig, - RunnerParms +import { + ArkTSEvolutionModule, + BuildType, + DeclFilesConfig, + DeclfileSideProductor, + DECLGEN_CACHE_FILE, + Params, + Payload, + ProjectConfig, + RunnerParms } from './type'; import fs from 'fs'; import path from 'path'; -import * as ts from 'typescript'; +import ts from 'typescript'; import { EXTNAME_D_ETS, EXTNAME_JS } from '../common/ark_define'; import { getRealModulePath } from '../../system_api/api_check_utils'; import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; import { calculateFileHash } from '../utils'; -import { runWithWorkerPool } from './thread_pool'; +// import { runWithWorkerPool } from './thread_pool'; +import { TaskDispatcher } from './2_thread_pool'; +// 2. trying export async function run(param: Params): Promise { FileManager.init(param.dependentModuleMap); DeclfileProductor.init(param); - const declgenTasks: ArkTSEvolutionModule[] = []; + const dispatcher = TaskDispatcher.getInstance(); + const tasks: Promise[] = []; - for (const task of param.tasks) { + param.tasks.forEach(task => { const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); + if (!moduleInfo || moduleInfo.dynamicFiles.length <= 0) { + return; + } if (task.buildTask === BuildType.DECLGEN) { - declgenTasks.push(moduleInfo); + const declfileSideProductor: DeclfileSideProductor = { + compilerOptions: DeclfileProductor.compilerOptions, + sdkConfigPrefix: DeclfileProductor.sdkConfigPrefix, + sdkConfigs: DeclfileProductor.sdkConfigs, + systemModules: DeclfileProductor.systemModules, + projectPath: DeclfileProductor.projectPath + }; + tasks.push( + dispatcher.runInThread( + moduleInfo, + declfileSideProductor + ) + ); } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); } + }); + + console.log('current tasks', tasks.length); + try { + await Promise.all(tasks); + return true; + } catch (error) { + console.error('Error in parallel task execution:', error); + return false; + } finally { + FileManager.cleanFileManagerObject(); } - - await runWithWorkerPool( - declgenTasks, - 4, - (moduleInfo) => DeclfileProductor.getInstance().getWorkerPayload(moduleInfo), - (moduleInfo, msg) => { - if (msg.status === 'error') { - console.error(`Declgen failed for ${moduleInfo.packageName}:`, msg.error); - } - } - ); - - FileManager.cleanFileManagerObject(); - return true; } +// 1. multiple threads version +// export async function run(param: Params): Promise { +// // console.log('something special'); +// FileManager.init(param.dependentModuleMap); +// DeclfileProductor.init(param); +// // console.log('stringify', JSON.stringify(param)); +// // console.log('stringifyMap', JSON.stringify(Object.fromEntries(param.dependentModuleMap.entries()))); + +// const declgenTasks: ArkTSEvolutionModule[] = []; + +// for (const task of param.tasks) { +// const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); +// if (task.buildTask === BuildType.DECLGEN) { +// declgenTasks.push(moduleInfo); +// } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { +// DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); +// } +// } + +// console.log('current tasks', declgenTasks.length); +// await runWithWorkerPool( +// declgenTasks, +// 4, +// (moduleInfo) => DeclfileProductor.getInstance().getWorkerPayload(moduleInfo), +// (moduleInfo, msg) => { +// if (msg.status === 'error') { +// console.error(`Declgen failed for ${moduleInfo.packageName}:`, msg.error); +// } +// } +// ); + +// FileManager.cleanFileManagerObject(); +// return true; +// } + +// 0. inittial version // export function run(param: Params): boolean { -// FileManager.init(param.dependentModuleMap); -// DeclfileProductor.init(param); -// param.tasks.forEach(task => { -// const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); -// if (moduleInfo?.dynamicFiles.length <= 0) { -// return; -// } -// if (task.buildTask === BuildType.DECLGEN) { -// DeclfileProductor.getInstance().runDeclgen(moduleInfo); -// } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { -// DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); -// } else if (task.buildTask === BuildType.BYTE_CODE_HAR) { -// //todo -// } -// }); -// FileManager.cleanFileManagerObject(); -// return true; +// FileManager.init(param.dependentModuleMap); +// DeclfileProductor.init(param); +// param.tasks.forEach(task => { +// const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); +// if (moduleInfo?.dynamicFiles.length <= 0) { +// return; +// } +// if (task.buildTask === BuildType.DECLGEN) { +// DeclfileProductor.getInstance().runDeclgen(moduleInfo); +// } else if (task.buildTask === BuildType.INTEROP_CONTEXT) { +// DeclfileProductor.getInstance().writeDeclFileInfo(moduleInfo, task.mainModuleName); +// } else if (task.buildTask === BuildType.BYTE_CODE_HAR) { +// // todo +// } +// }); +// FileManager.cleanFileManagerObject(); +// return true; // } -class DeclfileProductor { - private static declFileProductor: DeclfileProductor; - - static compilerOptions: ts.CompilerOptions; - static sdkConfigPrefix = 'ohos|system|kit|arkts'; - static sdkConfigs = []; - static systemModules = []; - static defaultSdkConfigs = []; - static projectPath; - private projectConfig; - private pkgDeclFilesConfig: { [pkgName: string]: DeclFilesConfig } = {}; - - static init(param: Params): void { - DeclfileProductor.declFileProductor = new DeclfileProductor(param); - DeclfileProductor.compilerOptions = ts.readConfigFile( - path.join(__dirname, '../../../../tsconfig.json'), ts.sys.readFile).config.compilerOptions; - DeclfileProductor.initSdkConfig(); - Object.assign(DeclfileProductor.compilerOptions, { - emitNodeModulesFiles: true, - importsNotUsedAsValues: ts.ImportsNotUsedAsValues.Preserve, - module: ts.ModuleKind.CommonJS, - moduleResolution: ts.ModuleResolutionKind.NodeJs, - noEmit: true, - packageManagerType: 'ohpm', - allowJs: true, - allowSyntheticDefaultImports: true, - esModuleInterop: true, - noImplicitAny: false, - noUnusedLocals: false, - noUnusedParameters: false, - experimentalDecorators: true, - resolveJsonModule: true, - skipLibCheck: false, - sourceMap: true, - target: 8, - types: [], - typeRoots: [], - lib: ['lib.es2021.d.ts'], - alwaysStrict: true, - checkJs: false, - maxFlowDepth: 2000, - etsAnnotationsEnable: false, - etsLoaderPath: path.join(__dirname, '../../../'), - needDoArkTsLinter: true, - isCompatibleVersion: false, - skipTscOhModuleCheck: false, - skipArkTSStaticBlocksCheck: false, - incremental: true, - tsImportSendableEnable: false, - skipPathsInKeyForCompilationSettings: true, - }); - DeclfileProductor.projectPath = param.projectConfig.projectRootPath; - } - static getInstance(param?: Params): DeclfileProductor { - if (!this.declFileProductor) { - this.declFileProductor = new DeclfileProductor(param); - } - return this.declFileProductor; +export class DeclfileProductor { + private static declFileProductor: DeclfileProductor; + + static compilerOptions: ts.CompilerOptions; + static sdkConfigPrefix = 'ohos|system|kit|arkts'; + static sdkConfigs = []; + static systemModules = []; + static defaultSdkConfigs = []; + static projectPath; + private projectConfig; + private pkgDeclFilesConfig: { [pkgName: string]: DeclFilesConfig } = {}; + + static init(param: Params): void { + DeclfileProductor.declFileProductor = new DeclfileProductor(param); + DeclfileProductor.compilerOptions = ts.readConfigFile( + path.join(__dirname, '../../../../tsconfig.json'), ts.sys.readFile).config.compilerOptions; + DeclfileProductor.initSdkConfig(); + Object.assign(DeclfileProductor.compilerOptions, { + emitNodeModulesFiles: true, + importsNotUsedAsValues: ts.ImportsNotUsedAsValues.Preserve, + module: ts.ModuleKind.CommonJS, + moduleResolution: ts.ModuleResolutionKind.NodeJs, + noEmit: true, + packageManagerType: 'ohpm', + allowJs: true, + allowSyntheticDefaultImports: true, + esModuleInterop: true, + noImplicitAny: false, + noUnusedLocals: false, + noUnusedParameters: false, + experimentalDecorators: true, + resolveJsonModule: true, + skipLibCheck: false, + sourceMap: true, + target: 8, + types: [], + typeRoots: [], + lib: ['lib.es2021.d.ts'], + alwaysStrict: true, + checkJs: false, + maxFlowDepth: 2000, + etsAnnotationsEnable: false, + etsLoaderPath: path.join(__dirname, '../../../'), + needDoArkTsLinter: true, + isCompatibleVersion: false, + skipTscOhModuleCheck: false, + skipArkTSStaticBlocksCheck: false, + incremental: true, + tsImportSendableEnable: false, + skipPathsInKeyForCompilationSettings: true + }); + DeclfileProductor.projectPath = param.projectConfig.projectRootPath; + } + static getInstance(param?: Params): DeclfileProductor { + if (!this.declFileProductor) { + this.declFileProductor = new DeclfileProductor(param); } + return this.declFileProductor; + } + + private constructor(param: Params) { + this.projectConfig = param.projectConfig as ProjectConfig; + } - private constructor(param: Params) { - this.projectConfig = param.projectConfig as ProjectConfig; + runDeclgen(moduleInfo: ArkTSEvolutionModule): void { + const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; + let existingCache = {}; + const filesToProcess = []; + + if (fs.existsSync(cachePath)) { + existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); } - runDeclgen(moduleInfo: ArkTSEvolutionModule): void { - const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; - let existingCache = {}; - const filesToProcess = []; - - if (fs.existsSync(cachePath)) { - existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); - } - - let inputList = []; - let hashMap = {}; - moduleInfo.dynamicFiles.forEach(path => { - let unixPath = toUnixPath(path); - const fileHash = calculateFileHash(path); - if (!existingCache[unixPath] || existingCache[unixPath] !== fileHash) { - filesToProcess.push(unixPath); - hashMap[unixPath] = fileHash; - } - }); - if (filesToProcess.length === 0) { - return; - } - readDeaclareFiles().forEach(path => { - filesToProcess.push(toUnixPath(path)); - }); - - inputList = inputList.filter(filePath => !filePath.endsWith('.js')); - const config: RunnerParms = { - inputDirs: [], - inputFiles: filesToProcess, - outDir: moduleInfo.declgenV2OutPath, - // use package name as folder name - rootDir: moduleInfo.modulePath, - customResolveModuleNames: resolveModuleNames, - customCompilerOptions: DeclfileProductor.compilerOptions, - includePaths: [moduleInfo.modulePath] - }; - if (!fs.existsSync(config.outDir)) { - fs.mkdirSync(config.outDir, { recursive: true }); - } - fs.mkdirSync(config.outDir, { recursive: true }); - generateInteropDecls(config); - processInteropUI(FileManager.arkTSModuleMap.get(moduleInfo.packageName)?.declgenV2OutPath); - const newCache = { - ...existingCache, - ...hashMap - }; - fs.writeFileSync(cachePath, JSON.stringify(newCache, null, 2)); + const hashMap = {}; + moduleInfo.dynamicFiles.forEach(path => { + const unixPath = toUnixPath(path); + const fileHash = calculateFileHash(path); + if (!existingCache[unixPath] || existingCache[unixPath] !== fileHash) { + filesToProcess.push(unixPath); + hashMap[unixPath] = fileHash; + } + }); + if (filesToProcess.length === 0) { + return; } + readDeaclareFiles().forEach(path => { + filesToProcess.push(toUnixPath(path)); + }); + + const config: RunnerParms = { + inputDirs: [], + inputFiles: filesToProcess, + outDir: moduleInfo.declgenV2OutPath, + // use package name as folder name + rootDir: moduleInfo.modulePath, + customResolveModuleNames: resolveModuleNames, + customCompilerOptions: DeclfileProductor.compilerOptions, + includePaths: [moduleInfo.modulePath] + }; + if (!fs.existsSync(config.outDir)) { + fs.mkdirSync(config.outDir, { recursive: true }); + } + fs.mkdirSync(config.outDir, { recursive: true }); + generateInteropDecls(config); + processInteropUI(FileManager.arkTSModuleMap.get(moduleInfo.packageName)?.declgenV2OutPath); + const newCache = { + ...existingCache, + ...hashMap + }; + fs.writeFileSync(cachePath, JSON.stringify(newCache, null, 2)); + } - writeDeclFileInfo(moduleInfo: ArkTSEvolutionModule, mainModuleName: string): void { - moduleInfo.dynamicFiles.forEach(file => { - this.addDeclFilesConfig(file, mainModuleName, this.projectConfig.bundleName, moduleInfo); - }); + writeDeclFileInfo(moduleInfo: ArkTSEvolutionModule, mainModuleName: string): void { + moduleInfo.dynamicFiles.forEach(file => { + this.addDeclFilesConfig(file, mainModuleName, this.projectConfig.bundleName, moduleInfo); + }); - const declFilesConfigFile: string = toUnixPath(moduleInfo.declFilesPath); - mkdirsSync(path.dirname(declFilesConfigFile)); - if (this.pkgDeclFilesConfig[moduleInfo.packageName]) { - fs.writeFileSync(declFilesConfigFile, JSON.stringify(this.pkgDeclFilesConfig[moduleInfo.packageName], null, 2), 'utf-8'); - } + const declFilesConfigFile: string = toUnixPath(moduleInfo.declFilesPath); + mkdirsSync(path.dirname(declFilesConfigFile)); + if (this.pkgDeclFilesConfig[moduleInfo.packageName]) { + fs.writeFileSync(declFilesConfigFile, JSON.stringify(this.pkgDeclFilesConfig[moduleInfo.packageName], null, 2), 'utf-8'); } + } + + addDeclFilesConfig(filePath: string, mainModuleName: string, bundleName: string, moduleInfo: ArkTSEvolutionModule): void { + const projectFilePath = getRelativePath(filePath, moduleInfo.modulePath); - addDeclFilesConfig(filePath: string, mainModuleName: string, bundleName: string, moduleInfo: ArkTSEvolutionModule): void { - const projectFilePath = getRelativePath(filePath, moduleInfo.modulePath); - - const declgenV2OutPath: string = this.getDeclgenV2OutPath(moduleInfo.packageName); - if (!declgenV2OutPath) { - return; - } - if (!this.pkgDeclFilesConfig[moduleInfo.packageName]) { - this.pkgDeclFilesConfig[moduleInfo.packageName] = { packageName: moduleInfo.packageName, files: {} }; - } - if (filePath.endsWith(EXTNAME_JS)) { - return; - } - if (this.pkgDeclFilesConfig[moduleInfo.packageName].files[projectFilePath]) { - return; - } - // The module name of the entry module of the project during the current compilation process. - const normalizedFilePath: string = `${moduleInfo.packageName}/${projectFilePath}`; - const declPath: string = path.join(toUnixPath(declgenV2OutPath), projectFilePath) + EXTNAME_D_ETS; - const ohmUrl: string = `N&${mainModuleName}&${bundleName}&${normalizedFilePath}&${moduleInfo.packageVersion}`; - this.pkgDeclFilesConfig[moduleInfo.packageName].files[projectFilePath] = { declPath, ohmUrl: `@normalized:${ohmUrl}` }; + const declgenV2OutPath: string = this.getDeclgenV2OutPath(moduleInfo.packageName); + if (!declgenV2OutPath) { + return; + } + if (!this.pkgDeclFilesConfig[moduleInfo.packageName]) { + this.pkgDeclFilesConfig[moduleInfo.packageName] = { packageName: moduleInfo.packageName, files: {} }; } + if (filePath.endsWith(EXTNAME_JS)) { + return; + } + if (this.pkgDeclFilesConfig[moduleInfo.packageName].files[projectFilePath]) { + return; + } + // The module name of the entry module of the project during the current compilation process. + const normalizedFilePath: string = `${moduleInfo.packageName}/${projectFilePath}`; + const declPath: string = path.join(toUnixPath(declgenV2OutPath), projectFilePath) + EXTNAME_D_ETS; + const ohmUrl: string = `N&${mainModuleName}&${bundleName}&${normalizedFilePath}&${moduleInfo.packageVersion}`; + this.pkgDeclFilesConfig[moduleInfo.packageName].files[projectFilePath] = { declPath, ohmUrl: `@normalized:${ohmUrl}` }; + } - getDeclgenV2OutPath(pkgName: string): string { - if (FileManager.arkTSModuleMap.size && FileManager.arkTSModuleMap.get(pkgName)) { - const arkTsModuleInfo: ArkTSEvolutionModule = FileManager.arkTSModuleMap.get(pkgName); - return arkTsModuleInfo.declgenV2OutPath; - } - return ''; + getDeclgenV2OutPath(pkgName: string): string { + if (FileManager.arkTSModuleMap.size && FileManager.arkTSModuleMap.get(pkgName)) { + const arkTsModuleInfo: ArkTSEvolutionModule = FileManager.arkTSModuleMap.get(pkgName); + return arkTsModuleInfo.declgenV2OutPath; } + return ''; + } - static initSdkConfig(): void { - const apiDirPath = path.resolve(__dirname, '../../../../../api'); - const arktsDirPath = path.resolve(__dirname, '../../../../../arkts'); - const kitsDirPath = path.resolve(__dirname, '../../../../../kits'); - const systemModulePathArray = [apiDirPath]; - if (!process.env.isFaMode) { - systemModulePathArray.push(arktsDirPath, kitsDirPath); - } - systemModulePathArray.forEach(systemModulesPath => { - if (fs.existsSync(systemModulesPath)) { - const modulePaths = []; - readFile(systemModulesPath, modulePaths); - DeclfileProductor.systemModules.push(...fs.readdirSync(systemModulesPath)); - const moduleSubdir = modulePaths.filter(filePath => { - const dirName = path.dirname(filePath); - return !(dirName === apiDirPath || dirName === arktsDirPath || dirName === kitsDirPath); - }).map(filePath => { - return filePath - .replace(apiDirPath, '') - .replace(arktsDirPath, '') - .replace(kitsDirPath, '') - .replace(/(^\\)|(.d.e?ts$)/g, '') - .replace(/\\/g, '/'); - }); - } - }); - DeclfileProductor.defaultSdkConfigs = [ - { - 'apiPath': systemModulePathArray, - 'prefix': '@ohos' - }, { - 'apiPath': systemModulePathArray, - 'prefix': '@system' - }, { - 'apiPath': systemModulePathArray, - 'prefix': '@arkts' - } - ]; - DeclfileProductor.sdkConfigs = [...DeclfileProductor.defaultSdkConfigs]; + static initSdkConfig(): void { + const apiDirPath = path.resolve(__dirname, '../../../../../api'); + const arktsDirPath = path.resolve(__dirname, '../../../../../arkts'); + const kitsDirPath = path.resolve(__dirname, '../../../../../kits'); + const systemModulePathArray = [apiDirPath]; + if (!process.env.isFaMode) { + systemModulePathArray.push(arktsDirPath, kitsDirPath); } + systemModulePathArray.forEach(systemModulesPath => { + if (fs.existsSync(systemModulesPath)) { + const modulePaths = []; + readFile(systemModulesPath, modulePaths); + DeclfileProductor.systemModules.push(...fs.readdirSync(systemModulesPath)); + } + }); + DeclfileProductor.defaultSdkConfigs = [ + { + 'apiPath': systemModulePathArray, + 'prefix': '@ohos' + }, { + 'apiPath': systemModulePathArray, + 'prefix': '@system' + }, { + 'apiPath': systemModulePathArray, + 'prefix': '@arkts' + } + ]; + DeclfileProductor.sdkConfigs = [...DeclfileProductor.defaultSdkConfigs]; + } + + getWorkerPayload(moduleInfo: ArkTSEvolutionModule): Payload { + const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; + let existingCache = {}; + const filesToProcess: string[] = []; + const hashMap = {}; - getWorkerPayload(moduleInfo: ArkTSEvolutionModule): any { - const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; - let existingCache = {}; - const filesToProcess: string[] = []; - const hashMap = {}; - - if (fs.existsSync(cachePath)) { - existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); - } - - moduleInfo.dynamicFiles.forEach(file => { - const unixPath = file.replace(/\\/g, '/'); - const hash = calculateFileHash(file); - if (!existingCache[unixPath] || existingCache[unixPath] !== hash) { - filesToProcess.push(unixPath); - hashMap[unixPath] = hash; - } - }); - - return { - inputDirs: [], - inputFiles: filesToProcess, - outDir: moduleInfo.declgenV2OutPath, - rootDir: moduleInfo.modulePath, - customResolveModuleNames: null, - customCompilerOptions: DeclfileProductor.compilerOptions, - includePaths: [moduleInfo.modulePath] - }; + if (fs.existsSync(cachePath)) { + existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); } + + moduleInfo.dynamicFiles.forEach(file => { + const unixPath = file.replace(/\\/g, '/'); + const hash = calculateFileHash(file); + if (!existingCache[unixPath] || existingCache[unixPath] !== hash) { + filesToProcess.push(unixPath); + hashMap[unixPath] = hash; + } + }); + + const newCache = { + ...existingCache, + ...hashMap + }; + return { + config: { + inputDirs: [], + inputFiles: filesToProcess, + outDir: moduleInfo.declgenV2OutPath, + rootDir: moduleInfo.modulePath, + customResolveModuleNames: null, + customCompilerOptions: DeclfileProductor.compilerOptions, + includePaths: [moduleInfo.modulePath] + }, + packageName: moduleInfo.packageName, cachePath, newCache}; + } } function resolveModuleNames(moduleNames: string[], containingFile: string): ts.ResolvedModuleFull[] { - const resolvedModules: ts.ResolvedModuleFull[] = []; - - for (const moduleName of moduleNames) { - let resolvedModule: ts.ResolvedModuleFull | null = null; - - resolvedModule = resolveWithDefault(moduleName, containingFile); - if (resolvedModule) { - resolvedModules.push(resolvedModule); - continue; - } - - resolvedModule = resolveSdkModule(moduleName); - if (resolvedModule) { - resolvedModules.push(resolvedModule); - continue; - } - - resolvedModule = resolveEtsModule(moduleName, containingFile); - if (resolvedModule) { - resolvedModules.push(resolvedModule); - continue; - } - - resolvedModule = resolveTsModule(moduleName, containingFile); - if (resolvedModule) { - resolvedModules.push(resolvedModule); - continue; - } - - resolvedModule = resolveOtherModule(moduleName, containingFile); - resolvedModules.push(resolvedModule ?? null); + const resolvedModules: ts.ResolvedModuleFull[] = []; + + for (const moduleName of moduleNames) { + let resolvedModule: ts.ResolvedModuleFull | null = null; + + resolvedModule = resolveWithDefault(moduleName, containingFile); + if (resolvedModule) { + resolvedModules.push(resolvedModule); + continue; + } + + resolvedModule = resolveSdkModule(moduleName); + if (resolvedModule) { + resolvedModules.push(resolvedModule); + continue; + } + + resolvedModule = resolveEtsModule(moduleName, containingFile); + if (resolvedModule) { + resolvedModules.push(resolvedModule); + continue; + } + + resolvedModule = resolveTsModule(moduleName, containingFile); + if (resolvedModule) { + resolvedModules.push(resolvedModule); + continue; } - return resolvedModules; + resolvedModule = resolveOtherModule(moduleName, containingFile); + resolvedModules.push(resolvedModule ?? null); + } + + return resolvedModules; } function resolveWithDefault( - moduleName: string, - containingFile: string + moduleName: string, + containingFile: string ): ts.ResolvedModuleFull | null { - const result = ts.resolveModuleName(moduleName, containingFile, DeclfileProductor.compilerOptions, moduleResolutionHost); - if (!result.resolvedModule) { - return null; - } + const result = ts.resolveModuleName(moduleName, containingFile, DeclfileProductor.compilerOptions, moduleResolutionHost); + if (!result.resolvedModule) { + return null; + } - const resolvedFileName = result.resolvedModule.resolvedFileName; - if (resolvedFileName && path.extname(resolvedFileName) === EXTNAME_JS) { - const resultDETSPath = resolvedFileName.replace(EXTNAME_JS, EXTNAME_D_ETS); - if (ts.sys.fileExists(resultDETSPath)) { - return getResolveModule(resultDETSPath, EXTNAME_D_ETS); - } + const resolvedFileName = result.resolvedModule.resolvedFileName; + if (resolvedFileName && path.extname(resolvedFileName) === EXTNAME_JS) { + const resultDETSPath = resolvedFileName.replace(EXTNAME_JS, EXTNAME_D_ETS); + if (ts.sys.fileExists(resultDETSPath)) { + return getResolveModule(resultDETSPath, EXTNAME_D_ETS); } + } - return result.resolvedModule; + return result.resolvedModule; } function resolveEtsModule(moduleName: string, containingFile: string): ts.ResolvedModuleFull | null { - if (!/\.ets$/.test(moduleName) || /\.d\.ets$/.test(moduleName)) { - return null; - } + if (!/\.ets$/.test(moduleName) || /\.d\.ets$/.test(moduleName)) { + return null; + } - const modulePath = path.resolve(path.dirname(containingFile), moduleName); - return ts.sys.fileExists(modulePath) ? getResolveModule(modulePath, '.ets') : null; + const modulePath = path.resolve(path.dirname(containingFile), moduleName); + return ts.sys.fileExists(modulePath) ? getResolveModule(modulePath, '.ets') : null; } function resolveSdkModule(moduleName: string): ts.ResolvedModuleFull | null { - const prefixRegex = new RegExp(`^@(${DeclfileProductor.sdkConfigPrefix})\\.`, 'i'); - if (!prefixRegex.test(moduleName.trim())) { - return null; - } + const prefixRegex = new RegExp(`^@(${DeclfileProductor.sdkConfigPrefix})\\.`, 'i'); + if (!prefixRegex.test(moduleName.trim())) { + return null; + } - for (const sdkConfig of DeclfileProductor.sdkConfigs) { - const resolveModuleInfo: ResolveModuleInfo = getRealModulePath(sdkConfig.apiPath, moduleName, ['.d.ts', '.d.ets']); - const modulePath: string = resolveModuleInfo.modulePath; - const isDETS: boolean = resolveModuleInfo.isEts; + for (const sdkConfig of DeclfileProductor.sdkConfigs) { + const resolveModuleInfo: ResolveModuleInfo = getRealModulePath(sdkConfig.apiPath, moduleName, ['.d.ts', '.d.ets']); + const modulePath: string = resolveModuleInfo.modulePath; + const isDETS: boolean = resolveModuleInfo.isEts; - const moduleKey = moduleName + (isDETS ? '.d.ets' : '.d.ts'); - if (DeclfileProductor.systemModules.includes(moduleKey) && ts.sys.fileExists(modulePath)) { - return getResolveModule(modulePath, isDETS ? '.d.ets' : '.d.ts'); - } + const moduleKey = moduleName + (isDETS ? '.d.ets' : '.d.ts'); + if (DeclfileProductor.systemModules.includes(moduleKey) && ts.sys.fileExists(modulePath)) { + return getResolveModule(modulePath, isDETS ? '.d.ets' : '.d.ts'); } + } - return null; + return null; } function resolveTsModule(moduleName: string, containingFile: string): ts.ResolvedModuleFull | null { - if (!/\.ts$/.test(moduleName)) { - return null; - } - + if (!/\.ts$/.test(moduleName)) { + return null; + } - const modulePath = path.resolve(path.dirname(containingFile), moduleName); - return ts.sys.fileExists(modulePath) ? getResolveModule(modulePath, '.ts') : null; + const modulePath = path.resolve(path.dirname(containingFile), moduleName); + return ts.sys.fileExists(modulePath) ? getResolveModule(modulePath, '.ts') : null; } function resolveOtherModule(moduleName: string, containingFile: string): ts.ResolvedModuleFull | null { - const apiModulePath = path.resolve(__dirname, '../../../api', moduleName + '.d.ts'); - const systemDETSModulePath = path.resolve(__dirname, '../../../api', moduleName + '.d.ets'); - const kitModulePath = path.resolve(__dirname, '../../../kits', moduleName + '.d.ts'); - const kitSystemDETSModulePath = path.resolve(__dirname, '../../../kits', moduleName + '.d.ets'); - const jsModulePath = path.resolve(__dirname, '../node_modules', moduleName + (moduleName.endsWith('.js') ? '' : '.js')); - const fileModulePath = path.resolve(__dirname, '../node_modules', moduleName + '/index.js'); - const DETSModulePath = path.resolve(path.dirname(containingFile), - moduleName.endsWith('.d.ets') ? moduleName : moduleName + EXTNAME_D_ETS); - - if (ts.sys.fileExists(apiModulePath)) { - return getResolveModule(apiModulePath, '.d.ts'); - } else if (ts.sys.fileExists(systemDETSModulePath)) { - return getResolveModule(systemDETSModulePath, '.d.ets'); - } else if (ts.sys.fileExists(kitModulePath)) { - return getResolveModule(kitModulePath, '.d.ts'); - } else if (ts.sys.fileExists(kitSystemDETSModulePath)) { - return getResolveModule(kitSystemDETSModulePath, '.d.ets'); - } else if (ts.sys.fileExists(jsModulePath)) { - return getResolveModule(jsModulePath, '.js'); - } else if (ts.sys.fileExists(fileModulePath)) { - return getResolveModule(fileModulePath, '.js'); - } else if (ts.sys.fileExists(DETSModulePath)) { - return getResolveModule(DETSModulePath, '.d.ets'); - } else { - const srcIndex = DeclfileProductor.projectPath.indexOf('src' + path.sep + 'main'); - if (srcIndex > 0) { - const DETSModulePathFromModule = path.resolve( - DeclfileProductor.projectPath.substring(0, srcIndex), - moduleName + path.sep + 'index' + EXTNAME_D_ETS - ); - if (ts.sys.fileExists(DETSModulePathFromModule)) { - return getResolveModule(DETSModulePathFromModule, '.d.ets'); - } - } - return null; + const apiModulePath = path.resolve(__dirname, '../../../api', moduleName + '.d.ts'); + const systemDETSModulePath = path.resolve(__dirname, '../../../api', moduleName + '.d.ets'); + const kitModulePath = path.resolve(__dirname, '../../../kits', moduleName + '.d.ts'); + const kitSystemDETSModulePath = path.resolve(__dirname, '../../../kits', moduleName + '.d.ets'); + const jsModulePath = path.resolve(__dirname, '../node_modules', moduleName + (moduleName.endsWith('.js') ? '' : '.js')); + const fileModulePath = path.resolve(__dirname, '../node_modules', moduleName + '/index.js'); + const DETSModulePath = path.resolve(path.dirname(containingFile), + moduleName.endsWith('.d.ets') ? moduleName : moduleName + EXTNAME_D_ETS); + + if (ts.sys.fileExists(apiModulePath)) { + return getResolveModule(apiModulePath, '.d.ts'); + } else if (ts.sys.fileExists(systemDETSModulePath)) { + return getResolveModule(systemDETSModulePath, '.d.ets'); + } else if (ts.sys.fileExists(kitModulePath)) { + return getResolveModule(kitModulePath, '.d.ts'); + } else if (ts.sys.fileExists(kitSystemDETSModulePath)) { + return getResolveModule(kitSystemDETSModulePath, '.d.ets'); + } else if (ts.sys.fileExists(jsModulePath)) { + return getResolveModule(jsModulePath, '.js'); + } else if (ts.sys.fileExists(fileModulePath)) { + return getResolveModule(fileModulePath, '.js'); + } else if (ts.sys.fileExists(DETSModulePath)) { + return getResolveModule(DETSModulePath, '.d.ets'); + } else { + const srcIndex = DeclfileProductor.projectPath.indexOf('src' + path.sep + 'main'); + if (srcIndex > 0) { + const DETSModulePathFromModule = path.resolve( + DeclfileProductor.projectPath.substring(0, srcIndex), + moduleName + path.sep + 'index' + EXTNAME_D_ETS + ); + if (ts.sys.fileExists(DETSModulePathFromModule)) { + return getResolveModule(DETSModulePathFromModule, '.d.ets'); + } } + return null; + } } function getRelativePath(filePath: string, pkgPath: string): string { - // rollup uses commonjs plugin to handle commonjs files, - // the commonjs files are prefixed with '\x00' and need to be removed. - if (filePath.startsWith('\x00')) { - filePath = filePath.replace('\x00', ''); - } - let unixFilePath: string = toUnixPath(filePath); - - // Handle .d.ets and .d.ts extensions - const dEtsIndex = unixFilePath.lastIndexOf('.d.ets'); - const dTsIndex = unixFilePath.lastIndexOf('.d.ts'); - - if (dEtsIndex !== -1) { - unixFilePath = unixFilePath.substring(0, dEtsIndex); - } else if (dTsIndex !== -1) { - unixFilePath = unixFilePath.substring(0, dTsIndex); - } else { - // Fallback to regular extension removal if not a .d file - const lastDotIndex = unixFilePath.lastIndexOf('.'); - if (lastDotIndex !== -1) { - unixFilePath = unixFilePath.substring(0, lastDotIndex); - } + // rollup uses commonjs plugin to handle commonjs files, + // the commonjs files are prefixed with '\x00' and need to be removed. + if (filePath.startsWith('\x00')) { + filePath = filePath.replace('\x00', ''); + } + let unixFilePath: string = toUnixPath(filePath); + + // Handle .d.ets and .d.ts extensions + const dEtsIndex = unixFilePath.lastIndexOf('.d.ets'); + const dTsIndex = unixFilePath.lastIndexOf('.d.ts'); + + if (dEtsIndex !== -1) { + unixFilePath = unixFilePath.substring(0, dEtsIndex); + } else if (dTsIndex !== -1) { + unixFilePath = unixFilePath.substring(0, dTsIndex); + } else { + // Fallback to regular extension removal if not a .d file + const lastDotIndex = unixFilePath.lastIndexOf('.'); + if (lastDotIndex !== -1) { + unixFilePath = unixFilePath.substring(0, lastDotIndex); } + } - const projectFilePath: string = unixFilePath.replace(toUnixPath(pkgPath) + '/', ''); - return projectFilePath; + const projectFilePath: string = unixFilePath.replace(toUnixPath(pkgPath) + '/', ''); + return projectFilePath; } const moduleResolutionHost: ts.ModuleResolutionHost = { - fileExists: (fileName: string): boolean => { - let exists = ts.sys.fileExists(fileName); - if (exists === undefined) { - exists = ts.sys.fileExists(fileName); - } - return exists; - }, - - readFile(fileName: string): string | undefined { - return ts.sys.readFile(fileName); - }, - realpath(path: string): string { - return ts.sys.realpath(path); - }, - trace(s: string): void { - console.info(s); + fileExists: (fileName: string): boolean => { + let exists = ts.sys.fileExists(fileName); + if (exists === undefined) { + exists = ts.sys.fileExists(fileName); + } + return exists; + }, + + readFile(fileName: string): string | undefined { + return ts.sys.readFile(fileName); + }, + realpath(path: string): string { + return ts.sys.realpath(path); + }, + trace(s: string): void { + console.info(s); + } +}; + +export function resolveModuleNamesByThread( + moduleNames: string[], + containingFile: string, + sideProductor: DeclfileSideProductor +): ts.ResolvedModuleFull[] { + const resolvedModules: ts.ResolvedModuleFull[] = []; + + for (const moduleName of moduleNames) { + let resolvedModule: ts.ResolvedModuleFull | null = null; + + resolvedModule = resolveWithDefaultByThread(moduleName, containingFile, sideProductor.compilerOptions); + if (resolvedModule) { + resolvedModules.push(resolvedModule); + continue; + } + + resolvedModule = resolveSdkModuleByThread(moduleName, sideProductor); + if (resolvedModule) { + resolvedModules.push(resolvedModule); + continue; + } + + resolvedModule = resolveEtsModule(moduleName, containingFile); + if (resolvedModule) { + resolvedModules.push(resolvedModule); + continue; + } + + resolvedModule = resolveTsModule(moduleName, containingFile); + if (resolvedModule) { + resolvedModules.push(resolvedModule); + continue; + } + + resolvedModule = resolveOtherModuleByThread(moduleName, containingFile, sideProductor); + resolvedModules.push(resolvedModule ?? null); + } + + return resolvedModules; +} + +function resolveWithDefaultByThread( + moduleName: string, + containingFile: string, + compilerOptions: ts.CompilerOptions +): ts.ResolvedModuleFull | null { + const result = ts.resolveModuleName(moduleName, containingFile, compilerOptions, moduleResolutionHost); + if (!result.resolvedModule) { + return null; + } + + const resolvedFileName = result.resolvedModule.resolvedFileName; + if (resolvedFileName && path.extname(resolvedFileName) === EXTNAME_JS) { + const resultDETSPath = resolvedFileName.replace(EXTNAME_JS, EXTNAME_D_ETS); + if (ts.sys.fileExists(resultDETSPath)) { + return getResolveModule(resultDETSPath, EXTNAME_D_ETS); } -}; \ No newline at end of file + } + + return result.resolvedModule; +} + +function resolveSdkModuleByThread( + moduleName: string, + sideProductor: DeclfileSideProductor +): ts.ResolvedModuleFull | null { + const prefixRegex = new RegExp(`^@(${sideProductor.sdkConfigPrefix})\\.`, 'i'); + if (!prefixRegex.test(moduleName.trim())) { + return null; + } + + for (const sdkConfig of sideProductor.sdkConfigs) { + const resolveModuleInfo: ResolveModuleInfo = getRealModulePath(sdkConfig.apiPath, moduleName, ['.d.ts', '.d.ets']); + const modulePath: string = resolveModuleInfo.modulePath; + const isDETS: boolean = resolveModuleInfo.isEts; + + const moduleKey = moduleName + (isDETS ? '.d.ets' : '.d.ts'); + if (sideProductor.systemModules.includes(moduleKey) && ts.sys.fileExists(modulePath)) { + return getResolveModule(modulePath, isDETS ? '.d.ets' : '.d.ts'); + } + } + + return null; +} + +function resolveOtherModuleByThread( + moduleName: string, + containingFile: string, + sideProductor: DeclfileSideProductor +): ts.ResolvedModuleFull | null { + const apiModulePath = path.resolve(__dirname, '../../../api', moduleName + '.d.ts'); + const systemDETSModulePath = path.resolve(__dirname, '../../../api', moduleName + '.d.ets'); + const kitModulePath = path.resolve(__dirname, '../../../kits', moduleName + '.d.ts'); + const kitSystemDETSModulePath = path.resolve(__dirname, '../../../kits', moduleName + '.d.ets'); + const jsModulePath = path.resolve(__dirname, '../node_modules', moduleName + (moduleName.endsWith('.js') ? '' : '.js')); + const fileModulePath = path.resolve(__dirname, '../node_modules', moduleName + '/index.js'); + const DETSModulePath = path.resolve(path.dirname(containingFile), + moduleName.endsWith('.d.ets') ? moduleName : moduleName + EXTNAME_D_ETS); + + if (ts.sys.fileExists(apiModulePath)) { + return getResolveModule(apiModulePath, '.d.ts'); + } else if (ts.sys.fileExists(systemDETSModulePath)) { + return getResolveModule(systemDETSModulePath, '.d.ets'); + } else if (ts.sys.fileExists(kitModulePath)) { + return getResolveModule(kitModulePath, '.d.ts'); + } else if (ts.sys.fileExists(kitSystemDETSModulePath)) { + return getResolveModule(kitSystemDETSModulePath, '.d.ets'); + } else if (ts.sys.fileExists(jsModulePath)) { + return getResolveModule(jsModulePath, '.js'); + } else if (ts.sys.fileExists(fileModulePath)) { + return getResolveModule(fileModulePath, '.js'); + } else if (ts.sys.fileExists(DETSModulePath)) { + return getResolveModule(DETSModulePath, '.d.ets'); + } else { + const srcIndex = sideProductor.projectPath.indexOf('src' + path.sep + 'main'); + if (srcIndex > 0) { + const DETSModulePathFromModule = path.resolve( + sideProductor.projectPath.substring(0, srcIndex), + moduleName + path.sep + 'index' + EXTNAME_D_ETS + ); + if (ts.sys.fileExists(DETSModulePathFromModule)) { + return getResolveModule(DETSModulePathFromModule, '.d.ets'); + } + } + return null; + } +} diff --git a/compiler/src/fast_build/ark_compiler/interop/test_all.ts b/compiler/src/fast_build/ark_compiler/interop/test_all.ts new file mode 100644 index 000000000..2061818d5 --- /dev/null +++ b/compiler/src/fast_build/ark_compiler/interop/test_all.ts @@ -0,0 +1,106 @@ +import { FileManager } from './interop_manager'; +import { readDeaclareFiles } from '../../../ets_checker'; +import { processInteropUI } from '../../../process_interop_ui'; +import { toUnixPath } from '../../../utils'; +import { + ArkTSEvolutionModule, + DeclFilesConfig, + DECLGEN_CACHE_FILE, + Params, + ProjectConfig, + RunnerParms +} from './type'; +import fs from 'fs'; +import ts from 'typescript'; +import { generateInteropDecls } from 'declgen/build/src/generateInteropDecls'; +import { calculateFileHash } from '../utils'; + +export function run(param: Params): boolean { + FileManager.init(param.dependentModuleMap); + param.tasks.forEach(task => { + const moduleInfo = FileManager.arkTSModuleMap.get(task.packageName); + if (moduleInfo?.dynamicFiles.length <= 0) { + return; + } + DeclfileProductor.getInstance().runDeclgen(moduleInfo); + }); + FileManager.cleanFileManagerObject(); + return true; +} + +class DeclfileProductor { + private static declFileProductor: DeclfileProductor; + + static compilerOptions: ts.CompilerOptions; + static sdkConfigPrefix = 'ohos|system|kit|arkts'; + static sdkConfigs = []; + static systemModules = []; + static defaultSdkConfigs = []; + static projectPath; + private projectConfig; + private pkgDeclFilesConfig: { [pkgName: string]: DeclFilesConfig } = {}; + + static getInstance(param?: Params): DeclfileProductor { + if (!this.declFileProductor) { + this.declFileProductor = new DeclfileProductor(param); + } + return this.declFileProductor; + } + + private constructor(param: Params) { + this.projectConfig = param.projectConfig as ProjectConfig; + } + + runDeclgen(moduleInfo: ArkTSEvolutionModule): void { + const cachePath = `${moduleInfo.declgenV2OutPath}/.${DECLGEN_CACHE_FILE}`; + let existingCache = {}; + const filesToProcess = []; + + if (fs.existsSync(cachePath)) { + existingCache = JSON.parse(fs.readFileSync(cachePath, 'utf-8')); + } + + const hashMap = {}; + moduleInfo.dynamicFiles.forEach(path => { + const unixPath = toUnixPath(path); + const fileHash = calculateFileHash(path); + if (!existingCache[unixPath] || existingCache[unixPath] !== fileHash) { + filesToProcess.push(unixPath); + hashMap[unixPath] = fileHash; + } + }); + if (filesToProcess.length === 0) { + return; + } + readDeaclareFiles().forEach(path => { + filesToProcess.push(toUnixPath(path)); + }); + + const config: RunnerParms = { + inputDirs: [], + inputFiles: filesToProcess, + outDir: moduleInfo.declgenV2OutPath, + // use package name as folder name + rootDir: moduleInfo.modulePath, + customResolveModuleNames: resolveModuleNames, + customCompilerOptions: DeclfileProductor.compilerOptions, + includePaths: [moduleInfo.modulePath] + }; + if (!fs.existsSync(config.outDir)) { + fs.mkdirSync(config.outDir, { recursive: true }); + } + fs.mkdirSync(config.outDir, { recursive: true }); + generateInteropDecls(config); + processInteropUI(FileManager.arkTSModuleMap.get(moduleInfo.packageName)?.declgenV2OutPath); + const newCache = { + ...existingCache, + ...hashMap + }; + fs.writeFileSync(cachePath, JSON.stringify(newCache, null, 2)); + } +} + +function resolveModuleNames(moduleNames: string[], containingFile: string): ts.ResolvedModuleFull[] { + const resolvedModules: ts.ResolvedModuleFull[] = []; + return resolvedModules; +} diff --git a/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts b/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts index 44830eac5..a8456a4bd 100644 --- a/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts +++ b/compiler/src/fast_build/ark_compiler/interop/thread_pool.ts @@ -1,5 +1,156 @@ import { Worker } from 'worker_threads'; -import path from 'path'; +import * as path from 'path'; + +interface WorkerTask { + task: T; + resolve: () => void; + reject: (reason?: any) => void; +} + +class WorkerPool { + private idleWorkers: Worker[] = []; + private activeWorkers = 0; + private taskQueue: WorkerTask[] = []; + private workerPath: string; + private getPayload: (task: T) => any; + private onMessage?: (task: T, msg: any) => void; + + constructor( + maxWorkers: number, + workerPath: string, + getPayload: (task: T) => any, + onMessage?: (task: T, msg: any) => void + ) { + this.workerPath = workerPath; + this.getPayload = getPayload; + this.onMessage = onMessage; + + // 预热线程池 + for (let i = 0; i < maxWorkers; i++) { + this.addWorker(); + } + } + + private addWorker() { + const worker = new Worker(this.workerPath); + worker.unref(); // 允许进程退出 + this.idleWorkers.push(worker); + } + + enqueue(task: T): Promise { + return new Promise((resolve, reject) => { + this.taskQueue.push({ task, resolve, reject }); + this.processQueue(); + }); + } + + private processQueue() { + while (this.idleWorkers.length > 0 && this.taskQueue.length > 0) { + const worker = this.idleWorkers.shift()!; + const taskItem = this.taskQueue.shift()!; + this.executeTask(worker, taskItem); + } + } + + private executeTask(worker: Worker, taskItem: WorkerTask) { + this.activeWorkers++; + + const { task, resolve, reject } = taskItem; + const payload = this.getPayload(task); + + // 清理旧监听器 + worker.removeAllListeners(); + + // 设置新监听器 + worker.on('message', (msg) => { + if (this.onMessage) { + this.onMessage(task, msg); + } + + if (msg.status === 'done') { + cleanup(); + resolve(); + this.returnWorker(worker); + } else if (msg.status === 'error') { + cleanup(); + reject(new Error(msg.error || 'Worker error')); + this.returnWorker(worker); + } + }); + + worker.on('error', (err) => { + cleanup(); + console.error(`Worker error: ${err.message}`); + reject(err); + this.replaceWorker(worker); + }); + + worker.on('exit', (code) => { + if (code !== 0) { + cleanup(); + reject(new Error(`Worker stopped with exit code ${code}`)); + this.replaceWorker(worker); + } + }); + + const cleanup = () => { + worker.removeAllListeners(); + this.activeWorkers--; + }; + + try { + worker.postMessage(payload); + } catch (err) { + cleanup(); + reject(err); + this.replaceWorker(worker); + } + } + + private returnWorker(worker: Worker) { + this.idleWorkers.push(worker); + this.processQueue(); + } + + private replaceWorker(worker: Worker) { + worker.terminate(); + this.addWorker(); + this.processQueue(); + } + + async drain() { + while (this.activeWorkers > 0 || this.taskQueue.length > 0) { + await new Promise(resolve => setTimeout(resolve, 100)); + } + } +} + +// export async function runWithWorkerPool( +// tasks: T[], +// maxConcurrency: number, +// getWorkerPayload: (task: T) => any, +// onWorkerMessage?: (task: T, msg: any) => void +// ): Promise { +// const workerPath = path.resolve(__dirname, './declgen_worker.js'); +// const pool = new WorkerPool( +// maxConcurrency, +// workerPath, +// getWorkerPayload, +// onWorkerMessage +// ); + +// // 添加任务到队列 +// const taskPromises = tasks.map(task => pool.enqueue(task)); +// try { +// await Promise.all(taskPromises); +// } catch (error) { +// console.error('Error in worker pool:', error); +// throw error; +// } finally { +// // 等待所有任务完成 +// await pool.drain(); +// } +// } /** * @param tasks 任务列表 @@ -7,20 +158,15 @@ import path from 'path'; * @param getWorkerPayload 将任务对象转换为传给 Worker 的数据 * @param onWorkerMessage 可选：处理 Worker 返回的消息 */ -export async function runWithWorkerPool( - tasks: T[], - maxConcurrency: number, - getWorkerPayload: (task: T) => any, - onWorkerMessage?: (task: T, msg: any) => void -): Promise { +export async function runWithWorkerPool(tasks: T[], maxConcurrency: number, getWorkerPayload: (task: T) => any, + onWorkerMessage?: (task: T, msg: any) => void): Promise { const queue = [...tasks]; const activeWorkers: Promise[] = []; - const runWorker = async (task: T): Promise => { + const runWorker = async(task: T): Promise => { return new Promise((resolve, reject) => { const workerPath = path.resolve(__dirname, './declgen_worker.js'); const worker = new Worker(workerPath); - const payload = getWorkerPayload(task); worker.postMessage(payload); @@ -29,29 +175,33 @@ export async function runWithWorkerPool( onWorkerMessage(task, msg); } if (msg.status === 'done') { + worker.terminate(); // 终止线程 resolve(); + // console.log('done and terminate'); } else if (msg.status === 'error') { reject(new Error(msg.error || 'Worker encountered an error')); + console.log('Worker encountered an error', msg.error); } }); - worker.on('error', (err) => { reject(err); + console.log('worker error', err); }); - worker.on('exit', (code) => { if (code !== 0) { reject(new Error(`Worker stopped with exit code ${code}`)); } + console.log('worker exit', code); }); }); }; for (let i = 0; i < maxConcurrency; i++) { - const loop = async () => { + const loop = async() => { while (queue.length > 0) { const task = queue.shift(); if (task) { + console.log('processing task'); await runWorker(task).catch((err) => { console.error(`[thread_pool] Worker task failed:`, err.message); }); @@ -60,6 +210,5 @@ export async function runWithWorkerPool( }; activeWorkers.push(loop()); } - await Promise.all(activeWorkers); } diff --git a/compiler/src/fast_build/ark_compiler/interop/type.ts b/compiler/src/fast_build/ark_compiler/interop/type.ts index 47387278d..cda0e885e 100644 --- a/compiler/src/fast_build/ark_compiler/interop/type.ts +++ b/compiler/src/fast_build/ark_compiler/interop/type.ts @@ -112,3 +112,18 @@ export interface FileInfo { abstractPath: string; } export const DECLGEN_CACHE_FILE = 'declgen_cache.json'; + +export interface Payload { + config: RunnerParms; + packageName: string; + cachePath: string; + newCache: {}; +} + +export interface DeclfileSideProductor { + compilerOptions: ts.CompilerOptions; + sdkConfigPrefix: string; + sdkConfigs: any[]; + systemModules: any[]; + projectPath: string; +} -- Gitee