diff --git a/ets2panda/driver/build_system/src/build/base_mode.ts b/ets2panda/driver/build_system/src/build/base_mode.ts index c3676d24ba5960433c02692660243800cfa6e8a1..eeb15985ea85f5a39b11127d44e3c6be469d4b31 100644 --- a/ets2panda/driver/build_system/src/build/base_mode.ts +++ b/ets2panda/driver/build_system/src/build/base_mode.ts @@ -83,7 +83,10 @@ import { RECORDE_RUN_NODE } from '../utils/record_time_mem'; import { TaskManager } from '../util/TaskManager'; -import { handleCompileWorkerExit } from '../util/worker_exit_handler'; +import { + handleCompileWorkerExit, + handleDeclgenWorkerExit +} from '../util/worker_exit_handler'; import { initKoalaModules } from '../init/init_koala_modules'; export abstract class BaseMode { @@ -1020,7 +1023,7 @@ export abstract class BaseMode { const taskManager = new TaskManager( path.resolve(__dirname, 'declgen_worker.js'), - handleCompileWorkerExit + handleDeclgenWorkerExit ); try { @@ -1034,7 +1037,7 @@ export abstract class BaseMode { }) ); - await Promise.all(taskPromises); + await Promise.allSettled(taskPromises); this.logger.printInfo('All declaration generation tasks complete.'); } catch (error) { diff --git a/ets2panda/driver/build_system/src/build/compile_worker.ts b/ets2panda/driver/build_system/src/build/compile_worker.ts index c43bf4d7f0a92e894902c6ed90ac68f5731191c7..fbb0b497c0fc5da6e0184e5495ae3583308cae21 100644 --- a/ets2panda/driver/build_system/src/build/compile_worker.ts +++ b/ets2panda/driver/build_system/src/build/compile_worker.ts @@ -105,7 +105,7 @@ process.on('message', async (message: { arkts.proceedToState(arkts.Es2pandaContextState.ES2PANDA_STATE_BIN_GENERATED, arktsGlobal.compilerContext.peer); if (process.send) { - process.send({ id, success: true }); + process.send({ id, success: true, shouldKill: false }); } } catch (error) { errorStatus = true; @@ -120,6 +120,7 @@ process.on('message', async (message: { process.send({ id, success: false, + shouldKill: true, error: serializeWithIgnore(logData) }); } diff --git a/ets2panda/driver/build_system/src/build/declgen_worker.ts b/ets2panda/driver/build_system/src/build/declgen_worker.ts index 6ed77c0556499e6eae3842e029168ca477d10931..fcb5ae0adca06c95140e87a8c0e3e75e55158674 100644 --- a/ets2panda/driver/build_system/src/build/declgen_worker.ts +++ b/ets2panda/driver/build_system/src/build/declgen_worker.ts @@ -15,13 +15,19 @@ import { CompileFileInfo, ModuleInfo } from '../types'; import { BuildConfig } from '../types'; -import { Logger } from '../logger'; +import { + Logger, + LogData, + LogDataFactory +} from '../logger'; +import { ErrorCode } from '../error_code'; import * as fs from 'fs'; import * as path from 'path'; import { changeDeclgenFileExtension, changeFileExtension, createFileIfNotExists, + serializeWithIgnore, ensurePathExists } from '../util/utils'; import { @@ -54,7 +60,8 @@ process.on('message', async (message: { pluginDriver.initPlugins(buildConfig); let { arkts, arktsGlobal } = initKoalaModules(buildConfig) - + let errorStatus = false; + let continueOnError = buildConfig.continueOnError ?? true; try { const source = fs.readFileSync(fileInfo.filePath, 'utf8'); const moduleInfo = moduleInfosMap.get(fileInfo.packageName)!; @@ -112,17 +119,25 @@ process.on('message', async (message: { logger.printInfo(`[declgen] ${fileInfo.filePath} processed successfully`); - process.send({ id, success: true }); + process.send({ id, success: true, shouldKill: false }); } catch (err) { + errorStatus = true; if (err instanceof Error) { + const logData: LogData = LogDataFactory.newInstance( + ErrorCode.BUILDSYSTEM_DECLGEN_FAIL, + 'Declgen generates declaration files failed.', + err.message, + fileInfo.filePath + ); process.send({ id, success: false, - error: `Generate declaration files failed.\n${err?.message || err}` + shouldKill: !continueOnError, + error: serializeWithIgnore(logData) }); } } finally { - if (arktsGlobal?.compilerContext?.peer) { + if (!errorStatus && arktsGlobal?.compilerContext?.peer) { arktsGlobal.es2panda._DestroyContext(arktsGlobal.compilerContext.peer); } if (arktsGlobal?.config) { diff --git a/ets2panda/driver/build_system/src/error_code.ts b/ets2panda/driver/build_system/src/error_code.ts index 86666dcf51166f69cd9a9155c41937522d467f75..529679ab833da155637873d5a8436ed7965ed581 100644 --- a/ets2panda/driver/build_system/src/error_code.ts +++ b/ets2panda/driver/build_system/src/error_code.ts @@ -45,4 +45,5 @@ export enum ErrorCode { BUILDSYSTEM_ALIAS_MODULE_PATH_NOT_EXIST = '11410024', BUILDSYSTEM_ENTRY_FILE_NOT_EXIST = "11410025", BUILDSYSTEM_COMPILE_FAILED_IN_WORKER = "11410026", + BUILDSYSTEM_DECLGEN_FAILED_IN_WORKER = '11410027', } diff --git a/ets2panda/driver/build_system/src/types.ts b/ets2panda/driver/build_system/src/types.ts index e7f74ebbb09296a86ba2e686030482d293b455fa..4730ac1e4bdff2bdd787893be5597dc18121b98b 100644 --- a/ets2panda/driver/build_system/src/types.ts +++ b/ets2panda/driver/build_system/src/types.ts @@ -172,6 +172,7 @@ export interface DeclgenConfig { declgenV2OutPath?: string; declgenBridgeCodePath?: string; skipDeclCheck?: boolean; + continueOnError?: boolean; } export interface LoggerConfig { diff --git a/ets2panda/driver/build_system/src/util/TaskManager.ts b/ets2panda/driver/build_system/src/util/TaskManager.ts index fafd1d3b87302c77bc58f98053fcdf7a11204ed0..7d9eec679ada4ad073e51f188524e1a25da457d2 100644 --- a/ets2panda/driver/build_system/src/util/TaskManager.ts +++ b/ets2panda/driver/build_system/src/util/TaskManager.ts @@ -25,6 +25,7 @@ export interface Task { payload: T; resolve: (result: true) => void; reject: (error: Object) => void; + timeoutTimer?: NodeJS.Timeout; } export interface WorkerInfo { @@ -37,12 +38,14 @@ export interface WorkerInfo { type OnWorkerExitCallback = ( workerInfo: WorkerInfo, code: number | null, + signal: NodeJS.Signals | null, runningTasks: Map> ) => void; interface WorkerMessage { id: string; success: boolean; + shouldKill: boolean; error?: LogData; } @@ -54,12 +57,15 @@ export class TaskManager { private maxWorkers = DEFAULT_WOKER_NUMS; private workerPath: string; private onWorkerExit: OnWorkerExitCallback; + private taskTimeoutMs: number; - constructor(workerPath: string, onWorkerExit: OnWorkerExitCallback, maxWorkers?: number) { + constructor(workerPath: string, onWorkerExit: OnWorkerExitCallback, + maxWorkers?: number, taskTimeoutMs: number = 180000) { const cpuCount = Math.max(os.cpus().length - 1, 1); this.workerPath = workerPath; this.onWorkerExit = onWorkerExit; + this.taskTimeoutMs = taskTimeoutMs; if (maxWorkers !== undefined) { this.maxWorkers = Math.min(maxWorkers, cpuCount); @@ -77,27 +83,11 @@ export class TaskManager { const workerInfo: WorkerInfo = { worker, id: i, isKilled: false }; worker.on('message', (message: WorkerMessage) => { - const { id, success, error } = message; - if (!success) { - this.shutdown(); - Logger.getInstance().printErrorAndExit(error!); - } - const task = this.runningTasks.get(id); - task?.resolve(true); - this.runningTasks.delete(id); - workerInfo.currentTaskId = undefined; - this.idleWorkers.push(workerInfo); - this.dispatchNext(); + this.handleWorkerMessage(workerInfo, message); }); - worker.on('exit', (code) => { - if (workerInfo.isKilled) { - return; - } - if (this.onWorkerExit) { - this.onWorkerExit(workerInfo, code, this.runningTasks); - return; - } + worker.on('exit', (code, signal) => { + this.handleWorkerExit(workerInfo, code, signal); }); this.workers.push(workerInfo); @@ -108,6 +98,109 @@ export class TaskManager { } + private settleTask(taskId: string, success: boolean, error?: string) { + const task = this.runningTasks.get(taskId); + if (!task) { + return; + } + if (task.timeoutTimer) { + clearTimeout(task.timeoutTimer); + task.timeoutTimer = undefined; + } + if (success) { + task.resolve(true); + } + else { + task.reject(error ?? new Error(error)); + } + this.runningTasks.delete(taskId); + } + + private handleSignals(workerInfo: WorkerInfo, signal: NodeJS.Signals | null) { + if (!signal) { + return; + } + switch (signal) { + case "SIGTERM": + break; + case "SIGSEGV": + this.reconfigureWorker(workerInfo); + break; + default: + break; + } + } + + private reconfigureWorker(workerInfo: WorkerInfo) { + const worker = fork(this.workerPath, [], { + stdio: ['inherit', 'inherit', 'inherit', 'ipc'] + }); + workerInfo.currentTaskId = undefined; + workerInfo.worker = worker; + worker.on('message', (message: WorkerMessage) => { + this.handleWorkerMessage(workerInfo, message); + }); + worker.on('exit', (code, signal) => { + this.handleWorkerExit(workerInfo, code, signal); + }); + this.idleWorkers.push(workerInfo); + } + + private handleWorkerExit(workerInfo: WorkerInfo, code: number | null, signal: NodeJS.Signals | null) { + const taskId = workerInfo.currentTaskId; + if (taskId) { + const success = code === 0 && !signal; + const reason = this.getWorkerExitReason(code, signal); + this.settleTask(taskId, success, reason); + } + + this.handleSignals(workerInfo, signal); + + if (this.onWorkerExit) { + this.onWorkerExit(workerInfo, code, signal, this.runningTasks); + } + } + + private logErrorMessage(message: WorkerMessage): void { + const err = message.error; + if (!err) { + return; + } + const logData = new LogData( + err.code, + err.description, + err.cause, + err.position, + err.solutions, + err.moreInfo + ); + if (message.shouldKill) { + this.shutdown(); + Logger.getInstance().printErrorAndExit(logData); + } else { + Logger.getInstance().printError(logData); + } + } + + private handleWorkerMessage(workerInfo: WorkerInfo, message: WorkerMessage) { + const { id, success } = message; + if (!success) { + this.logErrorMessage(message); + } + this.settleTask(id, success); + workerInfo.currentTaskId = undefined; + this.idleWorkers.push(workerInfo); + this.dispatchNext(); + } + + private getWorkerExitReason(code: number | null, signal: NodeJS.Signals | null): + string | undefined { + if (signal && signal !== 'SIGKILL') { + return `Worker killed by signal ${signal}`; + } + return code !== 0 ? `Worker exited with code ${code}` : undefined; + } + private dispatchNext(): void { while (this.taskQueue.length > 0 && this.idleWorkers.length > 0) { const task = this.taskQueue.shift()!; @@ -116,6 +209,14 @@ export class TaskManager { this.runningTasks.set(task.id, task); workerInfo.currentTaskId = task.id; + task.timeoutTimer = setTimeout(() => { + this.taskQueue.push(task); + workerInfo.currentTaskId = undefined; + workerInfo.worker.kill(); + this.reconfigureWorker(workerInfo); + this.dispatchNext(); + }, this.taskTimeoutMs); + workerInfo.worker.send({ id: task.id, payload: task.payload }); } } @@ -146,4 +247,4 @@ export class TaskManager { this.runningTasks.clear(); this.taskQueue = []; } -} \ No newline at end of file +} diff --git a/ets2panda/driver/build_system/src/util/worker_exit_handler.ts b/ets2panda/driver/build_system/src/util/worker_exit_handler.ts index 2378f29d297f53e956668681eb7ded91cac39d8b..13d798cf472ac6a4b523ab2b28b588c36fafc3c3 100644 --- a/ets2panda/driver/build_system/src/util/worker_exit_handler.ts +++ b/ets2panda/driver/build_system/src/util/worker_exit_handler.ts @@ -22,6 +22,7 @@ import { Task, WorkerInfo } from "./TaskManager"; export function handleCompileWorkerExit( workerInfo: WorkerInfo, code: number | null, + signal: NodeJS.Signals | null, runningTasks: Map> ): void { if (!code || code === 0) { @@ -51,3 +52,28 @@ export function handleCompileWorkerExit( Logger.getInstance().printErrorAndExit(logData); } +export function handleDeclgenWorkerExit( + workerInfo: WorkerInfo, + code: number | null, + signal: NodeJS.Signals | null, + runningTasks: Map> +): void { + + if (code && code !== 0) { + let logExitCodeData: LogData = LogDataFactory.newInstance( + ErrorCode.BUILDSYSTEM_DECLGEN_FAILED_IN_WORKER, + `Declgen crashed (exit code ${code})`, + "This error is likely caused internally from compiler.", + ); + Logger.getInstance().printError(logExitCodeData); + return; + } + if (signal && signal !== "SIGTERM") { + let logSignalData: LogData = LogDataFactory.newInstance( + ErrorCode.BUILDSYSTEM_DECLGEN_FAILED_IN_WORKER, + `Declgen crashed (exit signal ${signal})`, + "This error is likely caused internally from compiler.", + ); + Logger.getInstance().printError(logSignalData); + } +} diff --git a/ets2panda/driver/build_system/test/ut/declgen_workerTest/declgen_worker.test.ts b/ets2panda/driver/build_system/test/ut/declgen_workerTest/declgen_worker.test.ts index 5f392395710065ad5e9273d6fe9b68c624fde19c..7e57575fa7fcd701e9e1b4a2e3bcc450ac1cc069 100755 --- a/ets2panda/driver/build_system/test/ut/declgen_workerTest/declgen_worker.test.ts +++ b/ets2panda/driver/build_system/test/ut/declgen_workerTest/declgen_worker.test.ts @@ -20,13 +20,13 @@ jest.mock('path'); jest.mock('../../../src/util/utils', () => ({ // simplified functions for testing changeFileExtension: jest.fn((file: string, targetExt: string, originExt = '') => { - const currentExt = originExt.length === 0 ? file.substring(file.lastIndexOf('.')) - : originExt; + const currentExt = originExt.length === 0 ? file.substring(file.lastIndexOf('.')) + : originExt; const fileWithoutExt = file.substring(0, file.lastIndexOf(currentExt)); return fileWithoutExt + targetExt; }), changeDeclgenFileExtension: jest.fn((file: string, targetExt: string) => { - const DECL_ETS_SUFFIX = '.d.ets'; + const DECL_ETS_SUFFIX = '.d.ets'; if (file.endsWith(DECL_ETS_SUFFIX)) { return file.replace(DECL_ETS_SUFFIX, targetExt); } @@ -55,8 +55,11 @@ jest.mock('../../../src/logger', () => { } as any; return { Logger: mLogger, - LogDataFactory: { newInstance: jest.fn(() => ({ - code: '001', description: '', cause: '', position: '', solutions: [], moreInfo: {} })) } + LogDataFactory: { + newInstance: jest.fn(() => ({ + code: '001', description: '', cause: '', position: '', solutions: [], moreInfo: {} + })) + } }; }); jest.mock('../../../src/pre_define', () => ({ @@ -67,23 +70,23 @@ jest.mock('../../../src/pre_define', () => ({ jest.mock('../../../src/init/init_koala_modules', () => ({ initKoalaModules: jest.fn((buildConfig) => { - const fakeKoala = { - arkts: fakeArkts, - arktsGlobal: fakeArktsGlobal - }; - fakeKoala.arktsGlobal.es2panda._SetUpSoPath(buildConfig.pandaSdkPath); - - buildConfig.arkts = fakeKoala.arkts; - buildConfig.arktsGlobal = fakeKoala.arktsGlobal; - return fakeKoala; - }) + const fakeKoala = { + arkts: fakeArkts, + arktsGlobal: fakeArktsGlobal + }; + fakeKoala.arktsGlobal.es2panda._SetUpSoPath(buildConfig.pandaSdkPath); + + buildConfig.arkts = fakeKoala.arkts; + buildConfig.arktsGlobal = fakeKoala.arktsGlobal; + return fakeKoala; + }) })); const fakeArkts = { Config: { create: jest.fn(() => ({ peer: 'peer' })) }, - Context: { + Context: { createFromString: jest.fn(() => ({ program: {}, peer: 'peer' })), - createFromStringWithHistory: jest.fn(() => ({ program: {}, peer: 'peer' })) + createFromStringWithHistory: jest.fn(() => ({ program: {}, peer: 'peer' })) }, proceedToState: jest.fn(), Es2pandaContextState: { ES2PANDA_STATE_PARSED: 1, ES2PANDA_STATE_CHECKED: 2 }, @@ -124,14 +127,14 @@ afterEach(() => { // Test the functions of the declgen_worker.ts file describe('declgen_worker', () => { - const compileFileInfo ={ + const compileFileInfo = { filePath: '/src/foo.ets', dependentFiles: [], abcFilePath: 'foo.abc', arktsConfigFile: '/src/arktsconfig.json', packageName: 'pkg', }; - + const buildConfig = { hasMainModule: true, byteCodeHar: true, @@ -164,7 +167,7 @@ describe('declgen_worker', () => { dependenciesSet: new Set(), dependentSet: new Set(), }; - + const moduleInfos = [['pkg', moduleInfo]]; test('generate declaration && glue files && exit', () => { @@ -189,10 +192,11 @@ describe('declgen_worker', () => { expect(fakeArktsGlobal.es2panda._DestroyContext).toHaveBeenCalled(); expect(process.send).toHaveBeenCalledWith({ id: 'processId1', - success: true + success: true, + shouldKill: false }); }); - + test('destroy context && config', () => { require('fs').readFileSync.mockReturnValue('source code'); require('../../../src/build/declgen_worker'); @@ -206,5 +210,5 @@ describe('declgen_worker', () => { expect(fakeArkts.destroyConfig).toHaveBeenCalled(); expect(fakeArktsGlobal.es2panda._DestroyContext).toHaveBeenCalled(); }); - + });