From 894a2f9d035f4da0eda454e83d9e421d98fed202 Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Wed, 11 Jun 2025 20:07:22 +0300 Subject: [PATCH 01/14] WIP WIP WIP --- ui2abc/libarkts/package.json | 1 + ui2abc/libarkts/src-host/es2panda.ts | 124 ++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/ui2abc/libarkts/package.json b/ui2abc/libarkts/package.json index 207c451cc4..d26ad63c6c 100644 --- a/ui2abc/libarkts/package.json +++ b/ui2abc/libarkts/package.json @@ -59,6 +59,7 @@ "restart": "node ../fast-arktsc --config arktsconfig.json --compiler ../../incremental/tools/panda/arkts/ui2abc --link-name ./build/abc/main.abc --restart-stages && ninja -f build/abc/build.ninja", "direct": "node ../fast-arktsc --config arktsconfig.json --compiler ../../incremental/tools/panda/arkts/ui2abc --link-name ./build/abc/main.abc && ninja -f build/abc/build.ninja", "run": "npm run compile && npm run compile:plugins && npm run direct", + "es2panda:check": " rm -rf build/abc && npx rollup -c rollup.es2panda.mjs && npm run direct", "run:memo": "npm run compile && npm run compile:plugins && npm run compile --prefix ../memo-plugin && npm run memo", "run:abc": "$npm_package_config_panda_sdk_path/linux_host_tools/bin/ark --load-runtimes=ets --boot-panda-files=$npm_package_config_panda_sdk_path/ets/etsstdlib.abc ./main.abc main.ETSGLOBAL::main", "mocha": "PANDA_SDK_PATH=${PANDA_SDK_PATH:=$npm_package_config_panda_sdk_path} TS_NODE_PROJECT=./test/tsconfig.json mocha -r tsconfig-paths/register --reporter-option maxDiffSize=0", diff --git a/ui2abc/libarkts/src-host/es2panda.ts b/ui2abc/libarkts/src-host/es2panda.ts index c51a1ba16a..1fe58bac9c 100644 --- a/ui2abc/libarkts/src-host/es2panda.ts +++ b/ui2abc/libarkts/src-host/es2panda.ts @@ -15,12 +15,13 @@ import * as fs from "node:fs" import * as path from "node:path" -import { checkSDK, arktsGlobal as global, metaDatabase, runTransformer, runTransformerOnProgram, CompilationOptions, findStdlib } from "@koalaui/libarkts" +import { checkSDK, arktsGlobal as global, metaDatabase, runTransformer, CompilationOptions, findStdlib, passStringArray, passString } from "@koalaui/libarkts" import { CheckedBackFilter, PluginContextImpl } from "@koalaui/libarkts" import { Command } from "commander" import { filterSource, isNumber, throwError } from "@koalaui/libarkts" import { Es2pandaContextState } from "@koalaui/libarkts" -import { Options, AstNode, GlobalContext, Config, Context, createETSModuleFromContext, proceedToState, ProgramTransformer, rebindSubtree, recheckSubtree, dumpProgramInfo, dumpArkTsConfigInfo, collectDependencies } from "@koalaui/libarkts" +import { Options, AstNode, Config, Context, createETSModuleFromContext, proceedToState, ProgramTransformer, rebindSubtree, recheckSubtree, dumpProgramInfo, dumpArkTsConfigInfo } from "@koalaui/libarkts" +import { listPrograms, Program, defaultFilter } from "@koalaui/libarkts" function parseCommandLineArgs() { const commander = new Command() @@ -330,6 +331,11 @@ export function main() { const pluginContext = new PluginContextImpl() const pluginsByState = readAndSortPlugins(configDir, plugins) + if (!restartStages) { + withCache(configPath, files, outputs, pluginsByState) + return + } + for (var i = 0; i < files.length; i++) { invokeWithPlugins(configPath, files[i], outputs[i], pluginsByState, dumpAst, restartStages, stage, pluginNames, pluginContext) } @@ -341,3 +347,117 @@ function reportErrorAndExit(message: string): never { } main() + +interface ProgramInfo { + absoluteName: string, + isExternal: boolean, + depsCount: number, +} + +function withCache( + configPath: string, + inputs: string[], + outputs: string[], + pluginsByState: Map, +) { + const allFiles = inputs.flatMap(it => getAllDeps(configPath, it)) + const order = allFiles.map(it => { + return { absoluteName: it, isExternal: !inputs.includes(it), depsCount: getAllDeps(configPath, it).length } as ProgramInfo + }).sort((a: ProgramInfo, b: ProgramInfo) => a.depsCount - b.depsCount) // Good dependency resolving order! + const globalConfig = Config.create( + [ + '_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib(), + inputs[0] // Putting some file here to avoid "Fatal error: Unresolved module name" (should all files be in the same folder?) + ] + ) + const globalContext = global.es2panda._CreateGlobalContext( + globalConfig.peer, + passStringArray(allFiles), + allFiles.length, + false + ) + const actions: (() => void)[][] = [] + console.log(allFiles.length, order.length) + order.forEach((info: ProgramInfo) => { + const perProgramActions: (() => void)[] = [] + const args = ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib()] + if (!info.isExternal) { + const id = inputs.indexOf(info.absoluteName) + args.push('--output', outputs[id], inputs[id]) + } + const config = Config.create( + args + ) + console.log(info) + const context = global.es2panda._CreateCacheContextFromFile( + config.peer, + passString(info.absoluteName), + globalContext, + info.isExternal, + ) + console.log("cache context was created") + + /* ... apply plugins via per program actions ... */ + + perProgramActions.push(() => { + global.es2panda._ProceedToState( + context, + Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED + ) + console.log(global.es2panda._ContextState( + context + )) + }) + if (!info.isExternal) { + perProgramActions.push(() => { + global.es2panda._ProceedToState( + context, + Es2pandaContextState.ES2PANDA_STATE_BIN_GENERATED + ) + console.log(global.es2panda._ContextState( + context + )) + global.es2panda._DestroyConfig( + config.peer + ) + }) + } + actions.push(perProgramActions) + }) + for (let i = 0; ; i++) { + let actionWasTaken = false + for (let j = 0; j < actions.length; j++) { + if (i < actions[j].length) { + actions[j][i]() + actionWasTaken = true + } + } + if (!actionWasTaken) { + break + } + } +} + +function getAllDeps( + configPath: string, + file: string, +) { + const config = Config.create( + ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib(), file] + ) + const context = global.es2panda._CreateContextFromFile( + config.peer, + file, + ) + global.es2panda._ProceedToState( + context, + Es2pandaContextState.ES2PANDA_STATE_PARSED + ) + const programs = listPrograms( + new Program(global.es2panda._ContextProgram(context)), + defaultFilter, + context, + ) + global.compilerContext = new Context(context) + return programs.map(it => it.absoluteName) +} \ No newline at end of file -- Gitee From a68c020a4ade1a9450d021ca5ab51bfd30b1a90c Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Thu, 12 Jun 2025 15:14:21 +0300 Subject: [PATCH 02/14] runtime to ASM without plugin --- ui2abc/libarkts/package.json | 3 + ui2abc/libarkts/src-host/es2panda.ts | 94 ++++++++++++++++++++++------ 2 files changed, 77 insertions(+), 20 deletions(-) diff --git a/ui2abc/libarkts/package.json b/ui2abc/libarkts/package.json index d26ad63c6c..553d15f7c5 100644 --- a/ui2abc/libarkts/package.json +++ b/ui2abc/libarkts/package.json @@ -60,6 +60,9 @@ "direct": "node ../fast-arktsc --config arktsconfig.json --compiler ../../incremental/tools/panda/arkts/ui2abc --link-name ./build/abc/main.abc && ninja -f build/abc/build.ninja", "run": "npm run compile && npm run compile:plugins && npm run direct", "es2panda:check": " rm -rf build/abc && npx rollup -c rollup.es2panda.mjs && npm run direct", + "es2panda:check2": " rm -rf ../../incremental/common/build/abc && npx rollup -c rollup.es2panda.mjs && npm run build:common:inc:ui2abc --prefix ../../incremental/common", + "es2panda:check3": " rm -rf ../../incremental/runtime/build && npx rollup -c rollup.es2panda.mjs && npm run build:runtime:inc:ui2abc:recheck --prefix ../../incremental/runtime", + "es2panda:check4": " rm -rf ../../arkoala-arkts/arkui/build && npx rollup -c rollup.es2panda.mjs && npm run build:m3:recheck --prefix ../../arkoala-arkts/arkui", "run:memo": "npm run compile && npm run compile:plugins && npm run compile --prefix ../memo-plugin && npm run memo", "run:abc": "$npm_package_config_panda_sdk_path/linux_host_tools/bin/ark --load-runtimes=ets --boot-panda-files=$npm_package_config_panda_sdk_path/ets/etsstdlib.abc ./main.abc main.ETSGLOBAL::main", "mocha": "PANDA_SDK_PATH=${PANDA_SDK_PATH:=$npm_package_config_panda_sdk_path} TS_NODE_PROJECT=./test/tsconfig.json mocha -r tsconfig-paths/register --reporter-option maxDiffSize=0", diff --git a/ui2abc/libarkts/src-host/es2panda.ts b/ui2abc/libarkts/src-host/es2panda.ts index 1fe58bac9c..fad85f5bdf 100644 --- a/ui2abc/libarkts/src-host/es2panda.ts +++ b/ui2abc/libarkts/src-host/es2panda.ts @@ -15,7 +15,7 @@ import * as fs from "node:fs" import * as path from "node:path" -import { checkSDK, arktsGlobal as global, metaDatabase, runTransformer, CompilationOptions, findStdlib, passStringArray, passString } from "@koalaui/libarkts" +import { checkSDK, arktsGlobal as global, metaDatabase, runTransformer, CompilationOptions, findStdlib, passStringArray, passString, ETSModule } from "@koalaui/libarkts" import { CheckedBackFilter, PluginContextImpl } from "@koalaui/libarkts" import { Command } from "commander" import { filterSource, isNumber, throwError } from "@koalaui/libarkts" @@ -360,10 +360,15 @@ function withCache( outputs: string[], pluginsByState: Map, ) { - const allFiles = inputs.flatMap(it => getAllDeps(configPath, it)) + const timeStarted = new Date().getTime() + + const allFiles = [...new Set(inputs.flatMap(it => getAllDeps(configPath, it)))] + console.log(allFiles.length) const order = allFiles.map(it => { return { absoluteName: it, isExternal: !inputs.includes(it), depsCount: getAllDeps(configPath, it).length } as ProgramInfo }).sort((a: ProgramInfo, b: ProgramInfo) => a.depsCount - b.depsCount) // Good dependency resolving order! + console.log(allFiles.length) + const timeDepsResolved = new Date().getTime() const globalConfig = Config.create( [ '_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib(), @@ -376,8 +381,9 @@ function withCache( allFiles.length, false ) + + const timeGlobalsCreated = new Date().getTime() const actions: (() => void)[][] = [] - console.log(allFiles.length, order.length) order.forEach((info: ProgramInfo) => { const perProgramActions: (() => void)[] = [] const args = ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib()] @@ -388,35 +394,66 @@ function withCache( const config = Config.create( args ) - console.log(info) const context = global.es2panda._CreateCacheContextFromFile( config.peer, passString(info.absoluteName), globalContext, info.isExternal, ) - console.log("cache context was created") /* ... apply plugins via per program actions ... */ perProgramActions.push(() => { global.es2panda._ProceedToState( context, - Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED + Es2pandaContextState.ES2PANDA_STATE_PARSED ) - console.log(global.es2panda._ContextState( - context - )) }) + + perProgramActions.push(() => { + global.es2panda._ProceedToState( + context, + Es2pandaContextState.ES2PANDA_STATE_CHECKED + ) + }) + + perProgramActions.push(() => { + // How to exit checked stage on external programs? + // + // Going to lowered in deps results in errors like + // + // Fatal error: Failed to emit binary data: \ + // Base record std.core.Object$partial is not defined for record \ + // @koalaui/runtime.tree.TreeUpdater.TreeUpdater$partial + // + // Error above happends due to "export class" + // + // Recheck results in segfault (either main program or cached one) + }) + if (!info.isExternal) { perProgramActions.push(() => { + // If compiling more than 1 program, same error as descibed above happends global.es2panda._ProceedToState( context, - Es2pandaContextState.ES2PANDA_STATE_BIN_GENERATED + Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED + // Es2pandaContextState.ES2PANDA_STATE_LOWERED ) - console.log(global.es2panda._ContextState( + // const program = new Program(global.es2panda._ContextProgram(context)) + // const ast = new ETSModule(global.es2panda._ProgramAst(context, program.peer)) + // const src: string = ast.dumpSrc() + // const log = src.split('\n').map(it => `${program.fileName} | ${it}`).join('\n') + // console.log(log) + const state = global.es2panda._ContextState( context - )) + ) + console.log(Es2pandaContextState[state]) + global.es2panda._DestroyConfig( + config.peer + ) + }) + } else { + perProgramActions.push(() => { global.es2panda._DestroyConfig( config.peer ) @@ -424,6 +461,8 @@ function withCache( } actions.push(perProgramActions) }) + + const timeContextsAndTasksCreated = new Date().getTime() for (let i = 0; ; i++) { let actionWasTaken = false for (let j = 0; j < actions.length; j++) { @@ -436,6 +475,16 @@ function withCache( break } } + + const timeFinished = new Date().getTime() + + console.log( + `All Time: ${timeFinished - timeStarted}\n`, + `\tDeps: ${timeDepsResolved - timeStarted}\n`, + `\tGlobals: ${timeGlobalsCreated - timeDepsResolved}\n`, + `\tContextAndTasks: ${timeContextsAndTasksCreated - timeGlobalsCreated}\n`, + `\tTransformation: ${timeFinished - timeContextsAndTasksCreated}\n` + ) } function getAllDeps( @@ -445,19 +494,24 @@ function getAllDeps( const config = Config.create( ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib(), file] ) - const context = global.es2panda._CreateContextFromFile( - config.peer, - file, + const context = new Context( + global.es2panda._CreateContextFromFile( + config.peer, + file, + ) ) global.es2panda._ProceedToState( - context, + context.peer, Es2pandaContextState.ES2PANDA_STATE_PARSED ) const programs = listPrograms( - new Program(global.es2panda._ContextProgram(context)), + new Program(global.es2panda._ContextProgram(context.peer)), defaultFilter, - context, + context.peer, ) - global.compilerContext = new Context(context) - return programs.map(it => it.absoluteName) + global.compilerContext = context + const result = programs.map(it => it.absoluteName) + context.destroy() + config.destroy() + return result } \ No newline at end of file -- Gitee From 1cfb9010b71e03c7ffde8904b3d66de9820296ca Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Thu, 12 Jun 2025 17:18:59 +0300 Subject: [PATCH 03/14] wip --- arkoala-arkts/arkui/package.json | 2 +- incremental/runtime/package.json | 2 +- ui2abc/libarkts/src-host/es2panda.ts | 75 ++++++++++++++++++++++++++-- ui2abc/libarkts/src/plugin-utils.ts | 2 +- 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/arkoala-arkts/arkui/package.json b/arkoala-arkts/arkui/package.json index 9cb5413c04..afef9a31b3 100644 --- a/arkoala-arkts/arkui/package.json +++ b/arkoala-arkts/arkui/package.json @@ -10,7 +10,7 @@ "build:arkui-no-common": "npm run build:m3:recheck", "copy:abc": "mkdir -p ../build/bootPandaFiles/ && cp ./build-m3-impl/arkui-m3.abc ../build/bootPandaFiles/arkui.abc", "build:m3:restart": "node ../../ui2abc/fast-arktsc --config ./ui2abcconfig-m3.json --compiler ../../incremental/tools/panda/arkts/ui2abc --link-name ./build-m3-impl/arkui-m3.abc --restart-stages && ninja ${NINJA_OPTIONS} -f build-m3-impl/build.ninja -k 1200 && npm run copy:abc", - "build:m3:recheck": "node ../../ui2abc/fast-arktsc --config ./ui2abcconfig-recheck.json --compiler ../../incremental/tools/panda/arkts/ui2abc --link-name ./build-m3-impl/arkui-m3.abc && ninja ${NINJA_OPTIONS} -f build-m3-recheck/build.ninja -k 1200 && npm run copy:abc", + "build:m3:recheck": "node ../../ui2abc/fast-arktsc --config ./ui2abcconfig-recheck.json --compiler ../../incremental/tools/panda/arkts/ui2abc --link-name ./build-m3-impl/arkui-m3.abc --group-by 4 && ninja ${NINJA_OPTIONS} -f build-m3-recheck/build.ninja -k 1200 -j 1 && npm run copy:abc", "build:m3": "npm run build:m3:restart", "build:sdk-m3": "npm run build:sdk-m3:restart", "build:sdk-m3:restart": "node ../../ui2abc/fast-arktsc --config ./ui2abcconfig-sdk-m3.json --compiler ../../incremental/tools/panda/arkts/ui2abc --link-name ./build-m3-sdk/arkui-sdk.abc --restart-stages && ninja ${NINJA_OPTIONS} -f build-m3-sdk/build.ninja -k 1200", diff --git a/incremental/runtime/package.json b/incremental/runtime/package.json index 1a3f83f180..7d73567a4f 100644 --- a/incremental/runtime/package.json +++ b/incremental/runtime/package.json @@ -38,7 +38,7 @@ "annotate": "npm run compile --prefix ../../ui2abc/annotate && node ../../ui2abc/annotate", "fast-arktsc": "npm run compile --prefix ../../ui2abc/fast-arktsc", "build:runtime:inc:ui2abc": "npm run annotate && npm run fast-arktsc && node ../../ui2abc/fast-arktsc --config ./ui2abcconfig.json --compiler ../tools/panda/arkts/ui2abc --link-name ./build/runtime.abc --restart-stages --group-by 5 && PANDA_SDK_PATH=${PANDA_SDK_PATH:=../tools/panda/node_modules/@panda/sdk} ninja ${NINJA_OPTIONS} -f build/build.ninja", - "build:runtime:inc:ui2abc:recheck": "npm run annotate && npm run fast-arktsc && node ../../ui2abc/fast-arktsc --config ./ui2abcconfig-recheck.json --compiler ../tools/panda/arkts/ui2abc --link-name ./build/runtime.abc --group-by 5 && PANDA_SDK_PATH=${PANDA_SDK_PATH:=../tools/panda/node_modules/@panda/sdk} ninja ${NINJA_OPTIONS} -f build/recheck/build.ninja", + "build:runtime:inc:ui2abc:recheck": "npm run annotate && npm run fast-arktsc && node ../../ui2abc/fast-arktsc --config ./ui2abcconfig-recheck.json --compiler ../tools/panda/arkts/ui2abc --link-name ./build/runtime.abc --group-by 26 && PANDA_SDK_PATH=${PANDA_SDK_PATH:=../tools/panda/node_modules/@panda/sdk} ninja ${NINJA_OPTIONS} -f build/recheck/build.ninja", "build:runtime:with:tests": "npm run unmemoize:with:tests && fast-arktsc --config ./arktsconfig-test-unmemoized.json --compiler ../tools/panda/arkts/arktsc --link-name ./build/runtime-tests.abc && ninja ${NINJA_OPTIONS} -f build/unmemoized/abc/build.ninja", "build:incremental:components": "npm run build:compat && npm run build:common && npm run build:runtime", "build:incremental:components:inc": "npm run build:compat:inc && npm run build:common:inc && npm run build:runtime:inc", diff --git a/ui2abc/libarkts/src-host/es2panda.ts b/ui2abc/libarkts/src-host/es2panda.ts index fad85f5bdf..938216f95f 100644 --- a/ui2abc/libarkts/src-host/es2panda.ts +++ b/ui2abc/libarkts/src-host/es2panda.ts @@ -363,11 +363,9 @@ function withCache( const timeStarted = new Date().getTime() const allFiles = [...new Set(inputs.flatMap(it => getAllDeps(configPath, it)))] - console.log(allFiles.length) const order = allFiles.map(it => { return { absoluteName: it, isExternal: !inputs.includes(it), depsCount: getAllDeps(configPath, it).length } as ProgramInfo }).sort((a: ProgramInfo, b: ProgramInfo) => a.depsCount - b.depsCount) // Good dependency resolving order! - console.log(allFiles.length) const timeDepsResolved = new Date().getTime() const globalConfig = Config.create( [ @@ -410,6 +408,35 @@ function withCache( ) }) + perProgramActions.push(() => { + pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_PARSED)?.forEach(plugin => { + global.compilerContext = new Context(context) + if (info.isExternal) { + const emptyProgram = new Program(global.es2panda._ContextProgram(context)) + const programList = listPrograms(emptyProgram) + const thisProgam = programList.find(it => it.absoluteName == info.absoluteName) + if (!thisProgam) { + reportErrorAndExit(`No program ${info.absoluteName} in its cache context`) + } + runTransformer( + thisProgam, + Es2pandaContextState.ES2PANDA_STATE_PARSED, + false, + plugin, + new PluginContextImpl(), + ) + } else { + runTransformer( + new Program(global.es2panda._ContextProgram(context)), + Es2pandaContextState.ES2PANDA_STATE_PARSED, + false, + plugin, + new PluginContextImpl(), + ) + } + }) + }) + perProgramActions.push(() => { global.es2panda._ProceedToState( context, @@ -417,6 +444,47 @@ function withCache( ) }) + perProgramActions.push(() => { + pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_CHECKED)?.forEach(plugin => { + global.compilerContext = new Context(context) + if (info.isExternal) { + const emptyProgram = new Program(global.es2panda._ContextProgram(context)) + const programList = listPrograms(emptyProgram) + const thisProgam = programList.find(it => it.absoluteName == info.absoluteName) + if (!thisProgam) { + reportErrorAndExit(`No program ${info.absoluteName} in its cache context`) + } + runTransformer( + thisProgam, + Es2pandaContextState.ES2PANDA_STATE_CHECKED, + false, + plugin, + new PluginContextImpl(), + ) + } else { + runTransformer( + new Program(global.es2panda._ContextProgram(context)), + Es2pandaContextState.ES2PANDA_STATE_CHECKED, + false, + plugin, + new PluginContextImpl(), + ) + } + }) + }) + + perProgramActions.push(() => { + global.compilerContext = new Context(context) + global.config = config.peer + const program = global.es2panda._ContextProgram(context) + if (new Program(program).absoluteName == order[order.length - 1].absoluteName) { // ??? + console.log("RECHECK FOR", new Program(program).absoluteName) + recheckSubtree( + new ETSModule(global.es2panda._ProgramAst(context, program)) + ) + } + }) + perProgramActions.push(() => { // How to exit checked stage on external programs? // @@ -436,7 +504,8 @@ function withCache( // If compiling more than 1 program, same error as descibed above happends global.es2panda._ProceedToState( context, - Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED + Es2pandaContextState.ES2PANDA_STATE_BIN_GENERATED + // Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED // Es2pandaContextState.ES2PANDA_STATE_LOWERED ) // const program = new Program(global.es2panda._ContextProgram(context)) diff --git a/ui2abc/libarkts/src/plugin-utils.ts b/ui2abc/libarkts/src/plugin-utils.ts index f8903a4825..ebf836f343 100644 --- a/ui2abc/libarkts/src/plugin-utils.ts +++ b/ui2abc/libarkts/src/plugin-utils.ts @@ -95,7 +95,7 @@ export function runTransformer(prog: Program, state: Es2pandaContextState, resta runTransformerOnProgram(currentProgram, options, transform, pluginContext, hooks) // The first program is always the main program, so break here if should not proceed external sources - if (restart) break + break isMainProgram = false // Proceed to the next program -- Gitee From 3e802384569be9e60147c61c131c262ab652fa8f Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Mon, 16 Jun 2025 08:26:54 +0300 Subject: [PATCH 04/14] Add invalidate|remove|add file cache bridges --- ui2abc/libarkts/native/src/bridges.cc | 18 ++++++++++++++++++ ui2abc/libarkts/src/Es2pandaNativeModule.ts | 9 +++++++++ ui2abc/libarkts/src/arkts-api/peers/Context.ts | 12 ++++++++++++ 3 files changed, 39 insertions(+) diff --git a/ui2abc/libarkts/native/src/bridges.cc b/ui2abc/libarkts/native/src/bridges.cc index c57450bff3..92a0d87b3e 100644 --- a/ui2abc/libarkts/native/src/bridges.cc +++ b/ui2abc/libarkts/native/src/bridges.cc @@ -394,6 +394,24 @@ void impl_DestroyGlobalContext(KNativePointer globalContextPtr) { } KOALA_INTEROP_V1(DestroyGlobalContext, KNativePointer) +void impl_InvalidateFileCache(KNativePointer globalContextPtr, KStringPtr& fileName) { + auto globalContext = reinterpret_cast(globalContextPtr); + GetImpl()->InvalidateFileCache(globalContext, getStringCopy(fileName)); +} +KOALA_INTEROP_V2(InvalidateFileCache, KNativePointer, KStringPtr) + +void impl_RemoveFileCache(KNativePointer globalContextPtr, KStringPtr& fileName) { + auto globalContext = reinterpret_cast(globalContextPtr); + GetImpl()->RemoveFileCache(globalContext, getStringCopy(fileName)); +} +KOALA_INTEROP_V2(RemoveFileCache, KNativePointer, KStringPtr) + +void impl_AddFileCache(KNativePointer globalContextPtr, KStringPtr& fileName) { + auto globalContext = reinterpret_cast(globalContextPtr); + GetImpl()->AddFileCache(globalContext, getStringCopy(fileName)); +} +KOALA_INTEROP_V2(AddFileCache, KNativePointer, KStringPtr) + // From koala-wrapper // TODO check if some code should be generated diff --git a/ui2abc/libarkts/src/Es2pandaNativeModule.ts b/ui2abc/libarkts/src/Es2pandaNativeModule.ts index d64233b1ca..cb7437c36e 100644 --- a/ui2abc/libarkts/src/Es2pandaNativeModule.ts +++ b/ui2abc/libarkts/src/Es2pandaNativeModule.ts @@ -81,6 +81,15 @@ export class Es2pandaNativeModule { _DestroyGlobalContext(context: KNativePointer): void { throw new Error("Not implemented"); } + _InvalidateFileCache(globalContext: KNativePointer, fileName: String): void { + throw new Error("Not implemented"); + } + _RemoveFileCache(globalContext: KNativePointer, fileName: String): void { + throw new Error("Not implemented"); + } + _AddFileCache(globalContext: KNativePointer, fileName: String): void { + throw new Error("Not implemented"); + } _DestroyContext(context: KPtr): void { throw new Error("Not implemented") } diff --git a/ui2abc/libarkts/src/arkts-api/peers/Context.ts b/ui2abc/libarkts/src/arkts-api/peers/Context.ts index 349884d6e2..965a52fbe9 100644 --- a/ui2abc/libarkts/src/arkts-api/peers/Context.ts +++ b/ui2abc/libarkts/src/arkts-api/peers/Context.ts @@ -109,4 +109,16 @@ export class GlobalContext extends ArktsObject { this.peer = nullptr } } + + invalidateFileCache(fileName: string) { + global.es2panda._InvalidateFileCache(this.peer, fileName) + } + + removeFileCache(fileName: string) { + global.es2panda._RemoveFileCache(this.peer, fileName) + } + + addFileCache(fileName: string) { + global.es2panda._AddFileCache(this.peer, fileName) + } } -- Gitee From 3a44fd3b4e691278af6ad5f6d7a43d7c495412f5 Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Mon, 16 Jun 2025 16:37:25 +0300 Subject: [PATCH 05/14] WIP --- incremental/compat/src/arkts/observable.ts | 3 +- incremental/runtime/package.json | 2 +- incremental/runtime/src/memo/entry.ts | 5 + ui2abc/libarkts/src-host/es2panda.ts | 231 +++++++++------------ 4 files changed, 103 insertions(+), 138 deletions(-) diff --git a/incremental/compat/src/arkts/observable.ts b/incremental/compat/src/arkts/observable.ts index 7a6cd65b33..608f021330 100644 --- a/incremental/compat/src/arkts/observable.ts +++ b/incremental/compat/src/arkts/observable.ts @@ -249,7 +249,7 @@ function proxyChildrenOnly(array: T[], parent: ObservableHandler, observed?: array[i] = observableProxy(array[i], parent, observed) } } - +/* class ObservableArray extends Array { static $_invoke(array: Array, parent?: ObservableHandler, observed?: boolean): Array { return new ObservableArray(array, parent, observed); @@ -1085,3 +1085,4 @@ class ObservableDate extends Date { return super.setUTCMinutes(value, sec, ms) } } +*/ \ No newline at end of file diff --git a/incremental/runtime/package.json b/incremental/runtime/package.json index 7d73567a4f..fb49e6f0df 100644 --- a/incremental/runtime/package.json +++ b/incremental/runtime/package.json @@ -38,7 +38,7 @@ "annotate": "npm run compile --prefix ../../ui2abc/annotate && node ../../ui2abc/annotate", "fast-arktsc": "npm run compile --prefix ../../ui2abc/fast-arktsc", "build:runtime:inc:ui2abc": "npm run annotate && npm run fast-arktsc && node ../../ui2abc/fast-arktsc --config ./ui2abcconfig.json --compiler ../tools/panda/arkts/ui2abc --link-name ./build/runtime.abc --restart-stages --group-by 5 && PANDA_SDK_PATH=${PANDA_SDK_PATH:=../tools/panda/node_modules/@panda/sdk} ninja ${NINJA_OPTIONS} -f build/build.ninja", - "build:runtime:inc:ui2abc:recheck": "npm run annotate && npm run fast-arktsc && node ../../ui2abc/fast-arktsc --config ./ui2abcconfig-recheck.json --compiler ../tools/panda/arkts/ui2abc --link-name ./build/runtime.abc --group-by 26 && PANDA_SDK_PATH=${PANDA_SDK_PATH:=../tools/panda/node_modules/@panda/sdk} ninja ${NINJA_OPTIONS} -f build/recheck/build.ninja", + "build:runtime:inc:ui2abc:recheck": "npm run annotate && npm run fast-arktsc && node ../../ui2abc/fast-arktsc --config ./ui2abcconfig-recheck.json --compiler ../tools/panda/arkts/ui2abc --link-name ./build/runtime.abc --group-by 100 && PANDA_SDK_PATH=${PANDA_SDK_PATH:=../tools/panda/node_modules/@panda/sdk} ninja ${NINJA_OPTIONS} -f build/recheck/build.ninja", "build:runtime:with:tests": "npm run unmemoize:with:tests && fast-arktsc --config ./arktsconfig-test-unmemoized.json --compiler ../tools/panda/arkts/arktsc --link-name ./build/runtime-tests.abc && ninja ${NINJA_OPTIONS} -f build/unmemoized/abc/build.ninja", "build:incremental:components": "npm run build:compat && npm run build:common && npm run build:runtime", "build:incremental:components:inc": "npm run build:compat:inc && npm run build:common:inc && npm run build:runtime:inc", diff --git a/incremental/runtime/src/memo/entry.ts b/incremental/runtime/src/memo/entry.ts index b9389316ce..7107378881 100644 --- a/incremental/runtime/src/memo/entry.ts +++ b/incremental/runtime/src/memo/entry.ts @@ -89,3 +89,8 @@ export function memoEntry2( ): R { return entry(arg1, arg2) } + +/** @memo */ +export function foo() { + +} diff --git a/ui2abc/libarkts/src-host/es2panda.ts b/ui2abc/libarkts/src-host/es2panda.ts index 938216f95f..7997db7cb7 100644 --- a/ui2abc/libarkts/src-host/es2panda.ts +++ b/ui2abc/libarkts/src-host/es2panda.ts @@ -15,7 +15,7 @@ import * as fs from "node:fs" import * as path from "node:path" -import { checkSDK, arktsGlobal as global, metaDatabase, runTransformer, CompilationOptions, findStdlib, passStringArray, passString, ETSModule } from "@koalaui/libarkts" +import { checkSDK, arktsGlobal as global, metaDatabase, runTransformer, CompilationOptions, findStdlib, passStringArray, passString, ETSModule, recheckContext } from "@koalaui/libarkts" import { CheckedBackFilter, PluginContextImpl } from "@koalaui/libarkts" import { Command } from "commander" import { filterSource, isNumber, throwError } from "@koalaui/libarkts" @@ -365,7 +365,15 @@ function withCache( const allFiles = [...new Set(inputs.flatMap(it => getAllDeps(configPath, it)))] const order = allFiles.map(it => { return { absoluteName: it, isExternal: !inputs.includes(it), depsCount: getAllDeps(configPath, it).length } as ProgramInfo - }).sort((a: ProgramInfo, b: ProgramInfo) => a.depsCount - b.depsCount) // Good dependency resolving order! + }).sort((a: ProgramInfo, b: ProgramInfo) => { + if (a.isExternal == b.isExternal) + return a.depsCount - b.depsCount + if (a.isExternal && !b.isExternal) + return -1 + if (!a.isExternal && b.isExternal) + return 1 + throw new Error("never") + }) // Good dependency resolving order! const timeDepsResolved = new Date().getTime() const globalConfig = Config.create( [ @@ -381,10 +389,15 @@ function withCache( ) const timeGlobalsCreated = new Date().getTime() - const actions: (() => void)[][] = [] + order.forEach((info: ProgramInfo) => { - const perProgramActions: (() => void)[] = [] - const args = ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib()] + console.log(info.absoluteName, info.depsCount, info.isExternal) + }) + + console.log("============== COMPILATION STARTED ==============") + + order.forEach((info: ProgramInfo) => { + const args = ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib()] if (!info.isExternal) { const id = inputs.indexOf(info.absoluteName) args.push('--output', outputs[id], inputs[id]) @@ -398,161 +411,107 @@ function withCache( globalContext, info.isExternal, ) + + global.config = config.peer + global.compilerContext = new Context(context) - /* ... apply plugins via per program actions ... */ + console.log(info.absoluteName, info.depsCount, info.isExternal) - perProgramActions.push(() => { - global.es2panda._ProceedToState( - context, - Es2pandaContextState.ES2PANDA_STATE_PARSED - ) - }) + proceedToState(Es2pandaContextState.ES2PANDA_STATE_PARSED) - perProgramActions.push(() => { - pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_PARSED)?.forEach(plugin => { - global.compilerContext = new Context(context) - if (info.isExternal) { - const emptyProgram = new Program(global.es2panda._ContextProgram(context)) - const programList = listPrograms(emptyProgram) - const thisProgam = programList.find(it => it.absoluteName == info.absoluteName) - if (!thisProgam) { - reportErrorAndExit(`No program ${info.absoluteName} in its cache context`) - } - runTransformer( - thisProgam, - Es2pandaContextState.ES2PANDA_STATE_PARSED, - false, - plugin, - new PluginContextImpl(), - ) - } else { - runTransformer( - new Program(global.es2panda._ContextProgram(context)), - Es2pandaContextState.ES2PANDA_STATE_PARSED, - false, - plugin, - new PluginContextImpl(), - ) + pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_PARSED)?.forEach(plugin => { + if (info.isExternal) { + const emptyProgram = new Program(global.es2panda._ContextProgram(context)) + const programList = listPrograms(emptyProgram) + const thisProgram = programList.find(it => it.absoluteName == info.absoluteName) + if (!thisProgram) { + reportErrorAndExit(`No program ${info.absoluteName} in its cache context`) } - }) + runTransformer( + thisProgram, + Es2pandaContextState.ES2PANDA_STATE_PARSED, + false, + plugin, + new PluginContextImpl(), + ) + } else { + const program = new Program(global.es2panda._ContextProgram(context)) + runTransformer( + program, + Es2pandaContextState.ES2PANDA_STATE_PARSED, + false, + plugin, + new PluginContextImpl(), + ) + } }) - perProgramActions.push(() => { - global.es2panda._ProceedToState( - context, - Es2pandaContextState.ES2PANDA_STATE_CHECKED - ) - }) + proceedToState(Es2pandaContextState.ES2PANDA_STATE_CHECKED) - perProgramActions.push(() => { - pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_CHECKED)?.forEach(plugin => { - global.compilerContext = new Context(context) - if (info.isExternal) { - const emptyProgram = new Program(global.es2panda._ContextProgram(context)) - const programList = listPrograms(emptyProgram) - const thisProgam = programList.find(it => it.absoluteName == info.absoluteName) - if (!thisProgam) { - reportErrorAndExit(`No program ${info.absoluteName} in its cache context`) - } - runTransformer( - thisProgam, - Es2pandaContextState.ES2PANDA_STATE_CHECKED, - false, - plugin, - new PluginContextImpl(), - ) - } else { - runTransformer( - new Program(global.es2panda._ContextProgram(context)), - Es2pandaContextState.ES2PANDA_STATE_CHECKED, - false, - plugin, - new PluginContextImpl(), - ) + pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_CHECKED)?.forEach(plugin => { + if (info.isExternal) { + const emptyProgram = new Program(global.es2panda._ContextProgram(context)) + const programList = listPrograms(emptyProgram) + const thisProgram = programList.find(it => it.absoluteName == info.absoluteName) + if (!thisProgram) { + reportErrorAndExit(`No program ${info.absoluteName} in its cache context`) } - }) - }) - - perProgramActions.push(() => { - global.compilerContext = new Context(context) - global.config = config.peer - const program = global.es2panda._ContextProgram(context) - if (new Program(program).absoluteName == order[order.length - 1].absoluteName) { // ??? - console.log("RECHECK FOR", new Program(program).absoluteName) - recheckSubtree( - new ETSModule(global.es2panda._ProgramAst(context, program)) + runTransformer( + thisProgram, + Es2pandaContextState.ES2PANDA_STATE_CHECKED, + false, + plugin, + new PluginContextImpl(), + ) + } else { + const program = new Program(global.es2panda._ContextProgram(context)) + const programList = listPrograms(program) + const e = programList.find(it => it.absoluteName.includes("entry")) + console.log(e?.ast.dumpSrc()) + runTransformer( + program, + Es2pandaContextState.ES2PANDA_STATE_CHECKED, + false, + plugin, + new PluginContextImpl(), ) } }) - perProgramActions.push(() => { - // How to exit checked stage on external programs? - // - // Going to lowered in deps results in errors like - // - // Fatal error: Failed to emit binary data: \ - // Base record std.core.Object$partial is not defined for record \ - // @koalaui/runtime.tree.TreeUpdater.TreeUpdater$partial - // - // Error above happends due to "export class" - // - // Recheck results in segfault (either main program or cached one) - }) + console.log("RECHECK") + + recheckContext( + context + ) + + console.log("N") if (!info.isExternal) { - perProgramActions.push(() => { - // If compiling more than 1 program, same error as descibed above happends - global.es2panda._ProceedToState( - context, - Es2pandaContextState.ES2PANDA_STATE_BIN_GENERATED - // Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED - // Es2pandaContextState.ES2PANDA_STATE_LOWERED - ) - // const program = new Program(global.es2panda._ContextProgram(context)) - // const ast = new ETSModule(global.es2panda._ProgramAst(context, program.peer)) - // const src: string = ast.dumpSrc() - // const log = src.split('\n').map(it => `${program.fileName} | ${it}`).join('\n') - // console.log(log) - const state = global.es2panda._ContextState( - context - ) - console.log(Es2pandaContextState[state]) - global.es2panda._DestroyConfig( - config.peer - ) - }) + proceedToState(Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED) + // cannot go to BIN due to "export class" bug + // const program = new Program(global.es2panda._ContextProgram(context)) + // if (program.absoluteName?.includes("entry")) { + // console.log(program.ast.dumpSrc()) + // } } else { - perProgramActions.push(() => { - global.es2panda._DestroyConfig( - config.peer - ) - }) + proceedToState(Es2pandaContextState.ES2PANDA_STATE_LOWERED) } - actions.push(perProgramActions) + // TODO: why it breaks compilation? + // global.es2panda._DestroyContext( + // context + // ) + global.es2panda._DestroyConfig( + config.peer + ) }) - const timeContextsAndTasksCreated = new Date().getTime() - for (let i = 0; ; i++) { - let actionWasTaken = false - for (let j = 0; j < actions.length; j++) { - if (i < actions[j].length) { - actions[j][i]() - actionWasTaken = true - } - } - if (!actionWasTaken) { - break - } - } - const timeFinished = new Date().getTime() console.log( `All Time: ${timeFinished - timeStarted}\n`, `\tDeps: ${timeDepsResolved - timeStarted}\n`, `\tGlobals: ${timeGlobalsCreated - timeDepsResolved}\n`, - `\tContextAndTasks: ${timeContextsAndTasksCreated - timeGlobalsCreated}\n`, - `\tTransformation: ${timeFinished - timeContextsAndTasksCreated}\n` + `\tTransformation: ${timeFinished - timeGlobalsCreated}\n` ) } -- Gitee From ed6c9ec1fe3b29218c682b98be121a6755d0600e Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Tue, 17 Jun 2025 10:03:39 +0300 Subject: [PATCH 06/14] a bit stable --- incremental/runtime/src/memo/entry.ts | 5 - ui2abc/libarkts/native/src/bridges.cc | 18 -- ui2abc/libarkts/src-host/es2panda.ts | 183 +++++++++--------- ui2abc/libarkts/src/Es2pandaNativeModule.ts | 9 - .../libarkts/src/arkts-api/peers/Context.ts | 12 -- ui2abc/libarkts/src/plugin-utils.ts | 15 +- 6 files changed, 110 insertions(+), 132 deletions(-) diff --git a/incremental/runtime/src/memo/entry.ts b/incremental/runtime/src/memo/entry.ts index 7107378881..b9389316ce 100644 --- a/incremental/runtime/src/memo/entry.ts +++ b/incremental/runtime/src/memo/entry.ts @@ -89,8 +89,3 @@ export function memoEntry2( ): R { return entry(arg1, arg2) } - -/** @memo */ -export function foo() { - -} diff --git a/ui2abc/libarkts/native/src/bridges.cc b/ui2abc/libarkts/native/src/bridges.cc index 92a0d87b3e..c57450bff3 100644 --- a/ui2abc/libarkts/native/src/bridges.cc +++ b/ui2abc/libarkts/native/src/bridges.cc @@ -394,24 +394,6 @@ void impl_DestroyGlobalContext(KNativePointer globalContextPtr) { } KOALA_INTEROP_V1(DestroyGlobalContext, KNativePointer) -void impl_InvalidateFileCache(KNativePointer globalContextPtr, KStringPtr& fileName) { - auto globalContext = reinterpret_cast(globalContextPtr); - GetImpl()->InvalidateFileCache(globalContext, getStringCopy(fileName)); -} -KOALA_INTEROP_V2(InvalidateFileCache, KNativePointer, KStringPtr) - -void impl_RemoveFileCache(KNativePointer globalContextPtr, KStringPtr& fileName) { - auto globalContext = reinterpret_cast(globalContextPtr); - GetImpl()->RemoveFileCache(globalContext, getStringCopy(fileName)); -} -KOALA_INTEROP_V2(RemoveFileCache, KNativePointer, KStringPtr) - -void impl_AddFileCache(KNativePointer globalContextPtr, KStringPtr& fileName) { - auto globalContext = reinterpret_cast(globalContextPtr); - GetImpl()->AddFileCache(globalContext, getStringCopy(fileName)); -} -KOALA_INTEROP_V2(AddFileCache, KNativePointer, KStringPtr) - // From koala-wrapper // TODO check if some code should be generated diff --git a/ui2abc/libarkts/src-host/es2panda.ts b/ui2abc/libarkts/src-host/es2panda.ts index 7997db7cb7..47331eb109 100644 --- a/ui2abc/libarkts/src-host/es2panda.ts +++ b/ui2abc/libarkts/src-host/es2panda.ts @@ -15,13 +15,16 @@ import * as fs from "node:fs" import * as path from "node:path" -import { checkSDK, arktsGlobal as global, metaDatabase, runTransformer, CompilationOptions, findStdlib, passStringArray, passString, ETSModule, recheckContext } from "@koalaui/libarkts" +import { checkSDK, arktsGlobal as global, metaDatabase, runTransformer, CompilationOptions, findStdlib, passStringArray, passString, recheckContext, defaultFilter } from "@koalaui/libarkts" import { CheckedBackFilter, PluginContextImpl } from "@koalaui/libarkts" import { Command } from "commander" import { filterSource, isNumber, throwError } from "@koalaui/libarkts" import { Es2pandaContextState } from "@koalaui/libarkts" import { Options, AstNode, Config, Context, createETSModuleFromContext, proceedToState, ProgramTransformer, rebindSubtree, recheckSubtree, dumpProgramInfo, dumpArkTsConfigInfo } from "@koalaui/libarkts" -import { listPrograms, Program, defaultFilter } from "@koalaui/libarkts" +import { listPrograms, Program } from "@koalaui/libarkts" + +// This is to switch between adding stdlib to cache or not (probably makes no effect) +let withStdlib = false function parseCommandLineArgs() { const commander = new Command() @@ -362,9 +365,9 @@ function withCache( ) { const timeStarted = new Date().getTime() - const allFiles = [...new Set(inputs.flatMap(it => getAllDeps(configPath, it)))] + const allFiles = [...new Set(inputs.flatMap(it => getAllDeps(configPath, it, pluginsByState)))] const order = allFiles.map(it => { - return { absoluteName: it, isExternal: !inputs.includes(it), depsCount: getAllDeps(configPath, it).length } as ProgramInfo + return { absoluteName: it, isExternal: !inputs.includes(it), depsCount: getAllDeps(configPath, it, pluginsByState).length } as ProgramInfo }).sort((a: ProgramInfo, b: ProgramInfo) => { if (a.isExternal == b.isExternal) return a.depsCount - b.depsCount @@ -378,7 +381,7 @@ function withCache( const globalConfig = Config.create( [ '_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib(), - inputs[0] // Putting some file here to avoid "Fatal error: Unresolved module name" (should all files be in the same folder?) + inputs[0] // Putting some file here to avoid "Fatal error: Unresolved module name" ] ) const globalContext = global.es2panda._CreateGlobalContext( @@ -397,7 +400,7 @@ function withCache( console.log("============== COMPILATION STARTED ==============") order.forEach((info: ProgramInfo) => { - const args = ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib()] + const args = ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib()] if (!info.isExternal) { const id = inputs.indexOf(info.absoluteName) args.push('--output', outputs[id], inputs[id]) @@ -415,94 +418,80 @@ function withCache( global.config = config.peer global.compilerContext = new Context(context) - console.log(info.absoluteName, info.depsCount, info.isExternal) - - proceedToState(Es2pandaContextState.ES2PANDA_STATE_PARSED) - - pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_PARSED)?.forEach(plugin => { - if (info.isExternal) { - const emptyProgram = new Program(global.es2panda._ContextProgram(context)) - const programList = listPrograms(emptyProgram) - const thisProgram = programList.find(it => it.absoluteName == info.absoluteName) - if (!thisProgram) { - reportErrorAndExit(`No program ${info.absoluteName} in its cache context`) - } - runTransformer( - thisProgram, - Es2pandaContextState.ES2PANDA_STATE_PARSED, - false, - plugin, - new PluginContextImpl(), - ) - } else { + function applyPlugins(state: Es2pandaContextState) { + pluginsByState.get(state)?.forEach(plugin => { const program = new Program(global.es2panda._ContextProgram(context)) runTransformer( program, - Es2pandaContextState.ES2PANDA_STATE_PARSED, + state, false, plugin, new PluginContextImpl(), + {}, + false, + info.isExternal ? [info.absoluteName] : inputs ) - } - }) + }) + } - proceedToState(Es2pandaContextState.ES2PANDA_STATE_CHECKED) + function proceedToParsed() { + proceedToState(Es2pandaContextState.ES2PANDA_STATE_PARSED) + } - pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_CHECKED)?.forEach(plugin => { - if (info.isExternal) { - const emptyProgram = new Program(global.es2panda._ContextProgram(context)) - const programList = listPrograms(emptyProgram) - const thisProgram = programList.find(it => it.absoluteName == info.absoluteName) - if (!thisProgram) { - reportErrorAndExit(`No program ${info.absoluteName} in its cache context`) - } - runTransformer( - thisProgram, - Es2pandaContextState.ES2PANDA_STATE_CHECKED, - false, - plugin, - new PluginContextImpl(), - ) - } else { - const program = new Program(global.es2panda._ContextProgram(context)) - const programList = listPrograms(program) - const e = programList.find(it => it.absoluteName.includes("entry")) - console.log(e?.ast.dumpSrc()) - runTransformer( - program, - Es2pandaContextState.ES2PANDA_STATE_CHECKED, - false, - plugin, - new PluginContextImpl(), - ) - } - }) + function parsedPlugins() { + applyPlugins(Es2pandaContextState.ES2PANDA_STATE_PARSED) + } - console.log("RECHECK") + function proceedToChecked() { + proceedToState(Es2pandaContextState.ES2PANDA_STATE_CHECKED) + } - recheckContext( - context - ) + function checkedPlugins() { + applyPlugins(Es2pandaContextState.ES2PANDA_STATE_CHECKED) + } - console.log("N") + function recheck() { + recheckContext(context) + } - if (!info.isExternal) { - proceedToState(Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED) - // cannot go to BIN due to "export class" bug - // const program = new Program(global.es2panda._ContextProgram(context)) - // if (program.absoluteName?.includes("entry")) { - // console.log(program.ast.dumpSrc()) - // } - } else { - proceedToState(Es2pandaContextState.ES2PANDA_STATE_LOWERED) + function proceedToFinalState() { + if (!info.isExternal) { + proceedToState(Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED) + // cannot go to BIN due to "export class" bug + } else { + proceedToState(Es2pandaContextState.ES2PANDA_STATE_LOWERED) + } } - // TODO: why it breaks compilation? - // global.es2panda._DestroyContext( - // context - // ) - global.es2panda._DestroyConfig( - config.peer - ) + + function cleanup() { + // TODO: why it breaks anything? + // global.es2panda._DestroyContext( + // context + // ) + global.es2panda._DestroyConfig( + config.peer + ) + } + + const operations = [ + proceedToParsed, + parsedPlugins, + proceedToChecked, + checkedPlugins, + recheck, + proceedToFinalState, + cleanup, + ] + + console.log() + console.log(args.join(' ')) + console.log(info.absoluteName, info.depsCount, info.isExternal) + + operations.forEach(operation => { + console.log('>>>', operation.name) + operation() + console.log('>>>', operation.name, 'OK') + }) }) const timeFinished = new Date().getTime() @@ -518,9 +507,15 @@ function withCache( function getAllDeps( configPath: string, file: string, + pluginsByState: Map, ) { + const stdlibPath = path.normalize(findStdlib()) + if (file.includes(stdlibPath)) { + // No deps for stdlib, it should be processed before anything + return [] + } const config = Config.create( - ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib(), file] + ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', stdlibPath, file] ) const context = new Context( global.es2panda._CreateContextFromFile( @@ -528,18 +523,32 @@ function getAllDeps( file, ) ) - global.es2panda._ProceedToState( - context.peer, - Es2pandaContextState.ES2PANDA_STATE_PARSED - ) + global.config = config.peer + global.compilerContext = context + proceedToState(Es2pandaContextState.ES2PANDA_STATE_PARSED) + // We need to run parsed stage plugins because they might modify dependency lists + pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_PARSED)?.forEach(plugin => { + const program = new Program(global.es2panda._ContextProgram(context.peer)) + runTransformer( + program, + Es2pandaContextState.ES2PANDA_STATE_PARSED, + false, + plugin, + new PluginContextImpl(), + ) + }) + if (withStdlib) { // We want to put stdlib files to global context as well + proceedToState(Es2pandaContextState.ES2PANDA_STATE_CHECKED) + } + // TODO: build deps graph more optimally const programs = listPrograms( new Program(global.es2panda._ContextProgram(context.peer)), - defaultFilter, + withStdlib ? (_: string) => true : defaultFilter, context.peer, ) - global.compilerContext = context const result = programs.map(it => it.absoluteName) context.destroy() config.destroy() + withStdlib = false return result } \ No newline at end of file diff --git a/ui2abc/libarkts/src/Es2pandaNativeModule.ts b/ui2abc/libarkts/src/Es2pandaNativeModule.ts index cb7437c36e..d64233b1ca 100644 --- a/ui2abc/libarkts/src/Es2pandaNativeModule.ts +++ b/ui2abc/libarkts/src/Es2pandaNativeModule.ts @@ -81,15 +81,6 @@ export class Es2pandaNativeModule { _DestroyGlobalContext(context: KNativePointer): void { throw new Error("Not implemented"); } - _InvalidateFileCache(globalContext: KNativePointer, fileName: String): void { - throw new Error("Not implemented"); - } - _RemoveFileCache(globalContext: KNativePointer, fileName: String): void { - throw new Error("Not implemented"); - } - _AddFileCache(globalContext: KNativePointer, fileName: String): void { - throw new Error("Not implemented"); - } _DestroyContext(context: KPtr): void { throw new Error("Not implemented") } diff --git a/ui2abc/libarkts/src/arkts-api/peers/Context.ts b/ui2abc/libarkts/src/arkts-api/peers/Context.ts index 965a52fbe9..349884d6e2 100644 --- a/ui2abc/libarkts/src/arkts-api/peers/Context.ts +++ b/ui2abc/libarkts/src/arkts-api/peers/Context.ts @@ -109,16 +109,4 @@ export class GlobalContext extends ArktsObject { this.peer = nullptr } } - - invalidateFileCache(fileName: string) { - global.es2panda._InvalidateFileCache(this.peer, fileName) - } - - removeFileCache(fileName: string) { - global.es2panda._RemoveFileCache(this.peer, fileName) - } - - addFileCache(fileName: string) { - global.es2panda._AddFileCache(this.peer, fileName) - } } diff --git a/ui2abc/libarkts/src/plugin-utils.ts b/ui2abc/libarkts/src/plugin-utils.ts index ebf836f343..1e1335b69a 100644 --- a/ui2abc/libarkts/src/plugin-utils.ts +++ b/ui2abc/libarkts/src/plugin-utils.ts @@ -83,8 +83,18 @@ export function runTransformer(prog: Program, state: Es2pandaContextState, resta // The first program provided by program provider is the main program let currentProgram = provider.next() let isMainProgram = true + let pluginUsages = 0 while (currentProgram) { + // If explicit list of files to transform is specified, transform only them + if (explicitFileList && !explicitFileList.includes(currentProgram.absoluteName)) { + isMainProgram = false + currentProgram = provider.next() + continue + } + + pluginUsages++ + // Options passed to plugin and hooks const options: CompilationOptions = { isMainProgram, @@ -95,12 +105,15 @@ export function runTransformer(prog: Program, state: Es2pandaContextState, resta runTransformerOnProgram(currentProgram, options, transform, pluginContext, hooks) // The first program is always the main program, so break here if should not proceed external sources - break + if (restart) + break isMainProgram = false // Proceed to the next program currentProgram = provider.next() } + + console.log("PLUGIN USAGES:", pluginUsages) } function setAllParents(ast: AstNode) { -- Gitee From 12000e42b3ce2400c6a975d01946e45834b9eee2 Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Wed, 18 Jun 2025 20:56:35 +0300 Subject: [PATCH 07/14] Post rebase fixes --- ui2abc/libarkts/src-host/es2panda.ts | 129 +++---------------------- ui2abc/libarkts/src/arkts-api/index.ts | 1 + ui2abc/libarkts/src/index.ts | 1 + ui2abc/libarkts/src/plugin-utils.ts | 12 --- 4 files changed, 15 insertions(+), 128 deletions(-) diff --git a/ui2abc/libarkts/src-host/es2panda.ts b/ui2abc/libarkts/src-host/es2panda.ts index 47331eb109..7715cc4eac 100644 --- a/ui2abc/libarkts/src-host/es2panda.ts +++ b/ui2abc/libarkts/src-host/es2panda.ts @@ -21,7 +21,7 @@ import { Command } from "commander" import { filterSource, isNumber, throwError } from "@koalaui/libarkts" import { Es2pandaContextState } from "@koalaui/libarkts" import { Options, AstNode, Config, Context, createETSModuleFromContext, proceedToState, ProgramTransformer, rebindSubtree, recheckSubtree, dumpProgramInfo, dumpArkTsConfigInfo } from "@koalaui/libarkts" -import { listPrograms, Program } from "@koalaui/libarkts" +import { compileWithCache, listPrograms, Program } from "@koalaui/libarkts" // This is to switch between adding stdlib to cache or not (probably makes no effect) let withStdlib = false @@ -378,129 +378,26 @@ function withCache( throw new Error("never") }) // Good dependency resolving order! const timeDepsResolved = new Date().getTime() - const globalConfig = Config.create( - [ - '_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib(), - inputs[0] // Putting some file here to avoid "Fatal error: Unresolved module name" - ] - ) - const globalContext = global.es2panda._CreateGlobalContext( - globalConfig.peer, - passStringArray(allFiles), - allFiles.length, - false - ) - const timeGlobalsCreated = new Date().getTime() - - order.forEach((info: ProgramInfo) => { - console.log(info.absoluteName, info.depsCount, info.isExternal) - }) - - console.log("============== COMPILATION STARTED ==============") - - order.forEach((info: ProgramInfo) => { - const args = ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib()] - if (!info.isExternal) { - const id = inputs.indexOf(info.absoluteName) - args.push('--output', outputs[id], inputs[id]) - } - const config = Config.create( - args - ) - const context = global.es2panda._CreateCacheContextFromFile( - config.peer, - passString(info.absoluteName), - globalContext, - info.isExternal, - ) - - global.config = config.peer - global.compilerContext = new Context(context) - - function applyPlugins(state: Es2pandaContextState) { - pluginsByState.get(state)?.forEach(plugin => { - const program = new Program(global.es2panda._ContextProgram(context)) - runTransformer( - program, - state, - false, - plugin, - new PluginContextImpl(), - {}, - false, - info.isExternal ? [info.absoluteName] : inputs - ) - }) - } - - function proceedToParsed() { - proceedToState(Es2pandaContextState.ES2PANDA_STATE_PARSED) - } - - function parsedPlugins() { - applyPlugins(Es2pandaContextState.ES2PANDA_STATE_PARSED) - } - - function proceedToChecked() { - proceedToState(Es2pandaContextState.ES2PANDA_STATE_CHECKED) - } - - function checkedPlugins() { - applyPlugins(Es2pandaContextState.ES2PANDA_STATE_CHECKED) - } - - function recheck() { - recheckContext(context) - } - - function proceedToFinalState() { - if (!info.isExternal) { - proceedToState(Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED) - // cannot go to BIN due to "export class" bug - } else { - proceedToState(Es2pandaContextState.ES2PANDA_STATE_LOWERED) + compileWithCache( + configPath, + order.map(it => { + const idx = inputs.findIndex(it2 => it2 == it.absoluteName) + return { + absoluteName: it.absoluteName, + output: idx != -1 ? outputs[idx] : undefined, } - } - - function cleanup() { - // TODO: why it breaks anything? - // global.es2panda._DestroyContext( - // context - // ) - global.es2panda._DestroyConfig( - config.peer - ) - } - - const operations = [ - proceedToParsed, - parsedPlugins, - proceedToChecked, - checkedPlugins, - recheck, - proceedToFinalState, - cleanup, - ] - - console.log() - console.log(args.join(' ')) - console.log(info.absoluteName, info.depsCount, info.isExternal) - - operations.forEach(operation => { - console.log('>>>', operation.name) - operation() - console.log('>>>', operation.name, 'OK') - }) - }) + }), + pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_PARSED), + pluginsByState.get(Es2pandaContextState.ES2PANDA_STATE_CHECKED), + ) const timeFinished = new Date().getTime() console.log( `All Time: ${timeFinished - timeStarted}\n`, `\tDeps: ${timeDepsResolved - timeStarted}\n`, - `\tGlobals: ${timeGlobalsCreated - timeDepsResolved}\n`, - `\tTransformation: ${timeFinished - timeGlobalsCreated}\n` + `\tTransformation: ${timeFinished - timeDepsResolved}\n` ) } diff --git a/ui2abc/libarkts/src/arkts-api/index.ts b/ui2abc/libarkts/src/arkts-api/index.ts index e5497804db..2f968ab726 100644 --- a/ui2abc/libarkts/src/arkts-api/index.ts +++ b/ui2abc/libarkts/src/arkts-api/index.ts @@ -28,6 +28,7 @@ export * from "./plugins" export * from "./ImportStorage" export * from "./InferVoidReturnType" export * from "./ProgramProvider" +export * from "./CompileWithCache" export * from "./peers/AstNode" export * from "./peers/Config" diff --git a/ui2abc/libarkts/src/index.ts b/ui2abc/libarkts/src/index.ts index c1738ed5ea..64749e8c5b 100644 --- a/ui2abc/libarkts/src/index.ts +++ b/ui2abc/libarkts/src/index.ts @@ -33,6 +33,7 @@ export * from "./arkts-api/InferVoidReturnType" export * from "./arkts-api/ProgramProvider" export * from "./arkts-api/node-utilities/Program" export * from "./arkts-api/node-utilities/ArkTsConfig" +export * from "./arkts-api/CompileWithCache" export * from "./arkts-api/peers/AstNode" export * from "./arkts-api/peers/Config" diff --git a/ui2abc/libarkts/src/plugin-utils.ts b/ui2abc/libarkts/src/plugin-utils.ts index 1e1335b69a..97d3183da6 100644 --- a/ui2abc/libarkts/src/plugin-utils.ts +++ b/ui2abc/libarkts/src/plugin-utils.ts @@ -83,18 +83,8 @@ export function runTransformer(prog: Program, state: Es2pandaContextState, resta // The first program provided by program provider is the main program let currentProgram = provider.next() let isMainProgram = true - let pluginUsages = 0 while (currentProgram) { - // If explicit list of files to transform is specified, transform only them - if (explicitFileList && !explicitFileList.includes(currentProgram.absoluteName)) { - isMainProgram = false - currentProgram = provider.next() - continue - } - - pluginUsages++ - // Options passed to plugin and hooks const options: CompilationOptions = { isMainProgram, @@ -112,8 +102,6 @@ export function runTransformer(prog: Program, state: Es2pandaContextState, resta // Proceed to the next program currentProgram = provider.next() } - - console.log("PLUGIN USAGES:", pluginUsages) } function setAllParents(ast: AstNode) { -- Gitee From 810ee5683c6628a60b4c824a95fe43752bca5780 Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Wed, 18 Jun 2025 20:27:22 +0300 Subject: [PATCH 08/14] Improve cache pipeline --- .../src/arkts-api/CompileWithCache.ts | 60 ++++++++++++++----- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts b/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts index f25e23bb46..dbcaa5d9a7 100644 --- a/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts +++ b/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts @@ -21,7 +21,8 @@ import { passString, passStringArray } from "./utilities/private" import { Context } from "./peers/Context" import { Es2pandaContextState } from "../generated/Es2pandaEnums" import { Program } from "../generated" -import { runTransformer, runTransformerOnProgram } from "../plugin-utils" +import { runTransformerOnProgram } from "../plugin-utils" +import { ProgramProvider } from "./ProgramProvider" export interface ProgramInfo { /** @@ -34,6 +35,23 @@ export interface ProgramInfo { output: string | undefined } +export function getProgramsForPluginApplication( + programs: ProgramInfo[], +) { + const programsForCodeGeneration = programs + .filter(it => it.output != undefined) + .map(it => it.absoluteName) + // Generally, it would be ok to return just the list above, + // but for experimental purpose let's filter programs that + // are mentioned as both codegeneration and caching + const programsForCaching = new Set( + programs + .filter(it => it.output == undefined) + .map(it => it.absoluteName) + ) + return programsForCodeGeneration.filter(it => !programsForCaching.has(it)) +} + export function compileWithCache( configPath: string, orderedPrograms: ProgramInfo[], @@ -57,6 +75,10 @@ export function compileWithCache( false ) + const programsForPluginApplication = getProgramsForPluginApplication( + orderedPrograms + ) + orderedPrograms.forEach((it) => { const args = ['_', '--arktsconfig', configPath, '--extension', 'ets', '--stdlib', findStdlib()] if (it.output) { @@ -65,12 +87,12 @@ export function compileWithCache( const config = Config.create( args ) - const isMainProgram = (it.output != undefined) + const isExternalProgram = (it.output == undefined) const context = global.es2panda._CreateCacheContextFromFile( config.peer, passString(it.absoluteName), globalContext, - !isMainProgram, + isExternalProgram, ) global.config = config.peer @@ -79,20 +101,26 @@ export function compileWithCache( function applyPlugins(plugins: ProgramTransformer[] | undefined, state: Es2pandaContextState) { plugins?.forEach(plugin => { const program = new Program(global.es2panda._ContextProgram(context)) - const options: CompilationOptions = { - isMainProgram: isMainProgram, - stage: state, - restart: false + const provider = new ProgramProvider(program) + let currentProgram = provider.next() + let isFirstProgram = true + while (currentProgram) { + if (programsForPluginApplication.includes(currentProgram.absoluteName) || currentProgram.absoluteName == it.absoluteName) { + const options: CompilationOptions = { + isMainProgram: isFirstProgram, + stage: state, + restart: false, + } + runTransformerOnProgram( + currentProgram, + options, + plugin, + new PluginContextImpl(), + ) + } + currentProgram = provider.next() + isFirstProgram = false } - runTransformerOnProgram( - program, - options, - plugin, - new PluginContextImpl(), - {}, - // For external programs only that program (it is one of external programs in that context) - // For main programs all main programs - ) if (state == Es2pandaContextState.ES2PANDA_STATE_CHECKED) { recheckContext(context) } -- Gitee From cc2d62acc4cbcbd84d911f95d2278f230019405d Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Wed, 18 Jun 2025 21:06:49 +0300 Subject: [PATCH 09/14] sync --- ui2abc/libarkts/src/arkts-api/CompileWithCache.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts b/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts index dbcaa5d9a7..e7c53bde3c 100644 --- a/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts +++ b/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts @@ -145,16 +145,16 @@ export function compileWithCache( function proceedToFinalState() { if (it.output) { - proceedToState(Es2pandaContextState.ES2PANDA_STATE_BIN_GENERATED) + proceedToState(Es2pandaContextState.ES2PANDA_STATE_ASM_GENERATED) } else { proceedToState(Es2pandaContextState.ES2PANDA_STATE_LOWERED) } } function cleanup() { - global.es2panda._DestroyContext( - context - ) + // global.es2panda._DestroyContext( + // context + // ) global.es2panda._DestroyConfig( config.peer ) -- Gitee From f80727783732376cf008d32ea62cc5217abe6987 Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Wed, 18 Jun 2025 21:35:07 +0300 Subject: [PATCH 10/14] workaround for observable --- incremental/compat/src/arkts/observable.ts | 3 +-- ui2abc/libarkts/src-host/es2panda.ts | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/incremental/compat/src/arkts/observable.ts b/incremental/compat/src/arkts/observable.ts index 608f021330..7a6cd65b33 100644 --- a/incremental/compat/src/arkts/observable.ts +++ b/incremental/compat/src/arkts/observable.ts @@ -249,7 +249,7 @@ function proxyChildrenOnly(array: T[], parent: ObservableHandler, observed?: array[i] = observableProxy(array[i], parent, observed) } } -/* + class ObservableArray extends Array { static $_invoke(array: Array, parent?: ObservableHandler, observed?: boolean): Array { return new ObservableArray(array, parent, observed); @@ -1085,4 +1085,3 @@ class ObservableDate extends Date { return super.setUTCMinutes(value, sec, ms) } } -*/ \ No newline at end of file diff --git a/ui2abc/libarkts/src-host/es2panda.ts b/ui2abc/libarkts/src-host/es2panda.ts index 7715cc4eac..fc2b5920c4 100644 --- a/ui2abc/libarkts/src-host/es2panda.ts +++ b/ui2abc/libarkts/src-host/es2panda.ts @@ -369,6 +369,12 @@ function withCache( const order = allFiles.map(it => { return { absoluteName: it, isExternal: !inputs.includes(it), depsCount: getAllDeps(configPath, it, pluginsByState).length } as ProgramInfo }).sort((a: ProgramInfo, b: ProgramInfo) => { + if (a.absoluteName.includes("observable") && b.absoluteName.includes("observable")) + return 0 + if (a.absoluteName.includes("observable")) + return -99999 + if (b.absoluteName.includes("observable")) + return 99999 if (a.isExternal == b.isExternal) return a.depsCount - b.depsCount if (a.isExternal && !b.isExternal) -- Gitee From 2d43311c9372517ba00033667c6b4249cfe2ad85 Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Mon, 16 Jun 2025 08:26:54 +0300 Subject: [PATCH 11/14] [DO NOT MERGE] Do not visit block and chain expression due to cache bug --- incremental/common/src/math.ts | 2 +- .../src/arkts-api/SetAllParentsByPlugin.ts | 36 +++++++++++++++++++ ui2abc/libarkts/src/arkts-api/index.ts | 1 + ui2abc/libarkts/src/arkts-api/visitor.ts | 4 +++ ui2abc/libarkts/src/plugin-utils.ts | 5 +-- 5 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 ui2abc/libarkts/src/arkts-api/SetAllParentsByPlugin.ts diff --git a/incremental/common/src/math.ts b/incremental/common/src/math.ts index 8827d8d7b5..d3358f44af 100644 --- a/incremental/common/src/math.ts +++ b/incremental/common/src/math.ts @@ -68,7 +68,7 @@ export function parseNumber(str: string, name: string = "number", verify: boolea const value = asFloat64(str) if (verify) { const reverseStr = asString(value) - if (reverseStr !== undefined && reverseStr?.length == str.length && reverseStr == str) { + if (reverseStr !== undefined && reverseStr!.length == str.length && reverseStr == str) { return value } } diff --git a/ui2abc/libarkts/src/arkts-api/SetAllParentsByPlugin.ts b/ui2abc/libarkts/src/arkts-api/SetAllParentsByPlugin.ts new file mode 100644 index 0000000000..6afde46dd4 --- /dev/null +++ b/ui2abc/libarkts/src/arkts-api/SetAllParentsByPlugin.ts @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2022-2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { AbstractVisitor } from "./AbstractVisitor" +import { AstNode } from "./peers/AstNode" + +interface SetAllParentsOptions { + node: AstNode +} + +// We should not visit BlockExpression and ChainExpression in SetAllParents, so use handwritten SetAllParents +// See also changes in visitor.ts +class SetAllParents extends AbstractVisitor { + visitor(node: AstNode, options?: SetAllParentsOptions): AstNode { + if (options) { + node.parent = options.node + } + return this.visitEachChild(node, { node }) + } +} + +export function setAllParentsByPlugin(node: AstNode) { + new SetAllParents().visitor(node) +} diff --git a/ui2abc/libarkts/src/arkts-api/index.ts b/ui2abc/libarkts/src/arkts-api/index.ts index 2f968ab726..d2e22807f1 100644 --- a/ui2abc/libarkts/src/arkts-api/index.ts +++ b/ui2abc/libarkts/src/arkts-api/index.ts @@ -29,6 +29,7 @@ export * from "./ImportStorage" export * from "./InferVoidReturnType" export * from "./ProgramProvider" export * from "./CompileWithCache" +export * from "./SetAllParentsByPlugin" export * from "./peers/AstNode" export * from "./peers/Config" diff --git a/ui2abc/libarkts/src/arkts-api/visitor.ts b/ui2abc/libarkts/src/arkts-api/visitor.ts index f1fb3340f7..ac64f41d25 100644 --- a/ui2abc/libarkts/src/arkts-api/visitor.ts +++ b/ui2abc/libarkts/src/arkts-api/visitor.ts @@ -144,6 +144,10 @@ export function visitEachChild( node: AstNode, visitor: Visitor ): AstNode { + // Unable to visit such nodes due to es2panda cache bug + if (isBlockExpression(node) || isChainExpression(node)) { + return node + } global.profiler.nodeVisited() if (isETSModule(node)) { return updateETSModuleByStatements( diff --git a/ui2abc/libarkts/src/plugin-utils.ts b/ui2abc/libarkts/src/plugin-utils.ts index 97d3183da6..32187d1f0e 100644 --- a/ui2abc/libarkts/src/plugin-utils.ts +++ b/ui2abc/libarkts/src/plugin-utils.ts @@ -24,7 +24,8 @@ import { Program, inferVoidReturnType, ProgramProvider, - CompilationOptions + CompilationOptions, + setAllParentsByPlugin } from "./arkts-api" import { AstNode } from "./reexport-for-generated" @@ -70,7 +71,7 @@ export function runTransformerOnProgram(program: Program, options: CompilationOp importStorage.update() // Set parents of all nodes in AST - setAllParents(ast) + setAllParentsByPlugin(ast) // Perform some additional actions after the transformation end hooks.onProgramTransformEnd?.(options) -- Gitee From c7e8093ec095d8f0685b7032b596cbc55f99bd99 Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Thu, 19 Jun 2025 19:35:45 +0300 Subject: [PATCH 12/14] regenerate --- .../libarkts/native/src/generated/bridges.cc | 44 ++----------------- .../src/arkts-api/CompileWithCache.ts | 6 +-- .../src/generated/Es2pandaNativeModule.ts | 14 +----- ui2abc/libarkts/src/generated/index.ts | 1 - .../src/generated/peers/ScriptFunction.ts | 8 ---- .../libarkts/src/generated/peers/SrcDumper.ts | 7 +-- 6 files changed, 9 insertions(+), 71 deletions(-) diff --git a/ui2abc/libarkts/native/src/generated/bridges.cc b/ui2abc/libarkts/native/src/generated/bridges.cc index d8d866576f..681f980d0a 100644 --- a/ui2abc/libarkts/native/src/generated/bridges.cc +++ b/ui2abc/libarkts/native/src/generated/bridges.cc @@ -4254,25 +4254,6 @@ KUInt impl_ScriptFunctionFormalParamsLengthConst(KNativePointer context, KNative } KOALA_INTEROP_2(ScriptFunctionFormalParamsLengthConst, KUInt, KNativePointer, KNativePointer); -void impl_ScriptFunctionSetIsolatedDeclgenReturnType(KNativePointer context, KNativePointer receiver, KStringPtr& type) -{ - const auto _context = reinterpret_cast(context); - const auto _receiver = reinterpret_cast(receiver); - const auto _type = getStringCopy(type); - GetImpl()->ScriptFunctionSetIsolatedDeclgenReturnType(_context, _receiver, _type); - return ; -} -KOALA_INTEROP_V3(ScriptFunctionSetIsolatedDeclgenReturnType, KNativePointer, KNativePointer, KStringPtr); - -KNativePointer impl_ScriptFunctionGetIsolatedDeclgenReturnTypeConst(KNativePointer context, KNativePointer receiver) -{ - const auto _context = reinterpret_cast(context); - const auto _receiver = reinterpret_cast(receiver); - auto result = GetImpl()->ScriptFunctionGetIsolatedDeclgenReturnTypeConst(_context, _receiver); - return StageArena::strdup(result); -} -KOALA_INTEROP_2(ScriptFunctionGetIsolatedDeclgenReturnTypeConst, KNativePointer, KNativePointer, KNativePointer); - void impl_ScriptFunctionEmplaceReturnStatements(KNativePointer context, KNativePointer receiver, KNativePointer returnStatements) { const auto _context = reinterpret_cast(context); @@ -6718,15 +6699,6 @@ KNativePointer impl_AstNodeDumpDeclConst(KNativePointer context, KNativePointer } KOALA_INTEROP_2(AstNodeDumpDeclConst, KNativePointer, KNativePointer, KNativePointer); -KNativePointer impl_AstNodeIsolatedDumpDeclConst(KNativePointer context, KNativePointer receiver) -{ - const auto _context = reinterpret_cast(context); - const auto _receiver = reinterpret_cast(receiver); - auto result = GetImpl()->AstNodeIsolatedDumpDeclConst(_context, _receiver); - return StageArena::strdup(result); -} -KOALA_INTEROP_2(AstNodeIsolatedDumpDeclConst, KNativePointer, KNativePointer, KNativePointer); - void impl_AstNodeDumpConst(KNativePointer context, KNativePointer receiver, KNativePointer dumper) { const auto _context = reinterpret_cast(context); @@ -12058,16 +12030,15 @@ KNativePointer impl_CreateSrcDumper(KNativePointer context, KNativePointer node) } KOALA_INTEROP_2(CreateSrcDumper, KNativePointer, KNativePointer, KNativePointer); -KNativePointer impl_CreateSrcDumper1(KNativePointer context, KNativePointer node, KBoolean isDeclgen, KBoolean isIsolatedDeclgen) +KNativePointer impl_CreateSrcDumper1(KNativePointer context, KNativePointer node, KBoolean isDeclgen) { const auto _context = reinterpret_cast(context); const auto _node = reinterpret_cast(node); const auto _isDeclgen = static_cast(isDeclgen); - const auto _isIsolatedDeclgen = static_cast(isIsolatedDeclgen); - auto result = GetImpl()->CreateSrcDumper1(_context, _node, _isDeclgen, _isIsolatedDeclgen); + auto result = GetImpl()->CreateSrcDumper1(_context, _node, _isDeclgen); return result; } -KOALA_INTEROP_4(CreateSrcDumper1, KNativePointer, KNativePointer, KNativePointer, KBoolean, KBoolean); +KOALA_INTEROP_3(CreateSrcDumper1, KNativePointer, KNativePointer, KNativePointer, KBoolean); void impl_SrcDumperAdd(KNativePointer context, KNativePointer receiver, KStringPtr& str) { @@ -12185,15 +12156,6 @@ KBoolean impl_SrcDumperIsDeclgenConst(KNativePointer context, KNativePointer rec } KOALA_INTEROP_2(SrcDumperIsDeclgenConst, KBoolean, KNativePointer, KNativePointer); -KBoolean impl_SrcDumperIsIsolatedDeclgenConst(KNativePointer context, KNativePointer receiver) -{ - const auto _context = reinterpret_cast(context); - const auto _receiver = reinterpret_cast(receiver); - auto result = GetImpl()->SrcDumperIsIsolatedDeclgenConst(_context, _receiver); - return result; -} -KOALA_INTEROP_2(SrcDumperIsIsolatedDeclgenConst, KBoolean, KNativePointer, KNativePointer); - void impl_SrcDumperDumpNode(KNativePointer context, KNativePointer receiver, KStringPtr& key) { const auto _context = reinterpret_cast(context); diff --git a/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts b/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts index e7c53bde3c..689ff9a7b0 100644 --- a/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts +++ b/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts @@ -152,9 +152,9 @@ export function compileWithCache( } function cleanup() { - // global.es2panda._DestroyContext( - // context - // ) + global.es2panda._DestroyContext( + context + ) global.es2panda._DestroyConfig( config.peer ) diff --git a/ui2abc/libarkts/src/generated/Es2pandaNativeModule.ts b/ui2abc/libarkts/src/generated/Es2pandaNativeModule.ts index 833446f05e..db1f2d58e2 100644 --- a/ui2abc/libarkts/src/generated/Es2pandaNativeModule.ts +++ b/ui2abc/libarkts/src/generated/Es2pandaNativeModule.ts @@ -1336,12 +1336,6 @@ export class Es2pandaNativeModule { _ScriptFunctionFormalParamsLengthConst(context: KNativePointer, receiver: KNativePointer): KUInt { throw new Error("This methods was not overloaded by native module initialization") } - _ScriptFunctionSetIsolatedDeclgenReturnType(context: KNativePointer, receiver: KNativePointer, type: KStringPtr): void { - throw new Error("This methods was not overloaded by native module initialization") - } - _ScriptFunctionGetIsolatedDeclgenReturnTypeConst(context: KNativePointer, receiver: KNativePointer): KStringPtr { - throw new Error("This methods was not overloaded by native module initialization") - } _ScriptFunctionEmplaceReturnStatements(context: KNativePointer, receiver: KNativePointer, returnStatements: KNativePointer): void { throw new Error("This methods was not overloaded by native module initialization") } @@ -2098,9 +2092,6 @@ export class Es2pandaNativeModule { _AstNodeDumpDeclConst(context: KNativePointer, receiver: KNativePointer): KStringPtr { throw new Error("This methods was not overloaded by native module initialization") } - _AstNodeIsolatedDumpDeclConst(context: KNativePointer, receiver: KNativePointer): KStringPtr { - throw new Error("This methods was not overloaded by native module initialization") - } _AstNodeDumpConst(context: KNativePointer, receiver: KNativePointer, dumper: KNativePointer): void { throw new Error("This methods was not overloaded by native module initialization") } @@ -3742,7 +3733,7 @@ export class Es2pandaNativeModule { _CreateSrcDumper(context: KNativePointer, node: KNativePointer): KNativePointer { throw new Error("This methods was not overloaded by native module initialization") } - _CreateSrcDumper1(context: KNativePointer, node: KNativePointer, isDeclgen: KBoolean, isIsolatedDeclgen: KBoolean): KNativePointer { + _CreateSrcDumper1(context: KNativePointer, node: KNativePointer, isDeclgen: KBoolean): KNativePointer { throw new Error("This methods was not overloaded by native module initialization") } _SrcDumperAdd(context: KNativePointer, receiver: KNativePointer, str: KStringPtr): void { @@ -3781,9 +3772,6 @@ export class Es2pandaNativeModule { _SrcDumperIsDeclgenConst(context: KNativePointer, receiver: KNativePointer): KBoolean { throw new Error("This methods was not overloaded by native module initialization") } - _SrcDumperIsIsolatedDeclgenConst(context: KNativePointer, receiver: KNativePointer): KBoolean { - throw new Error("This methods was not overloaded by native module initialization") - } _SrcDumperDumpNode(context: KNativePointer, receiver: KNativePointer, key: KStringPtr): void { throw new Error("This methods was not overloaded by native module initialization") } diff --git a/ui2abc/libarkts/src/generated/index.ts b/ui2abc/libarkts/src/generated/index.ts index acb52f8f42..8ee50a1a66 100644 --- a/ui2abc/libarkts/src/generated/index.ts +++ b/ui2abc/libarkts/src/generated/index.ts @@ -208,7 +208,6 @@ export * from "./peers/NewExpression" export * from "./peers/TSParameterProperty" export * from "./peers/ETSWildcardType" export * from "./peers/TSThisType" -export * from "./peers/ETSDynamicFunctionType" export * from "./peers/InterfaceDecl" export * from "./peers/FunctionDecl" export * from "./peers/Program" diff --git a/ui2abc/libarkts/src/generated/peers/ScriptFunction.ts b/ui2abc/libarkts/src/generated/peers/ScriptFunction.ts index 4cd3a14437..6ca88b1836 100644 --- a/ui2abc/libarkts/src/generated/peers/ScriptFunction.ts +++ b/ui2abc/libarkts/src/generated/peers/ScriptFunction.ts @@ -190,14 +190,6 @@ export class ScriptFunction extends AstNode { return global.generatedEs2panda._ScriptFunctionFormalParamsLengthConst(global.context, this.peer) } /** @deprecated */ - setIsolatedDeclgenReturnType(type: string): this { - global.generatedEs2panda._ScriptFunctionSetIsolatedDeclgenReturnType(global.context, this.peer, type) - return this - } - get isolatedDeclgenReturnType(): string { - return unpackString(global.generatedEs2panda._ScriptFunctionGetIsolatedDeclgenReturnTypeConst(global.context, this.peer)) - } - /** @deprecated */ emplaceReturnStatements(returnStatements?: ReturnStatement): this { global.generatedEs2panda._ScriptFunctionEmplaceReturnStatements(global.context, this.peer, passNode(returnStatements)) return this diff --git a/ui2abc/libarkts/src/generated/peers/SrcDumper.ts b/ui2abc/libarkts/src/generated/peers/SrcDumper.ts index 8ec090e542..ff56dbb497 100644 --- a/ui2abc/libarkts/src/generated/peers/SrcDumper.ts +++ b/ui2abc/libarkts/src/generated/peers/SrcDumper.ts @@ -32,8 +32,8 @@ export class SrcDumper extends ArktsObject { constructor(pointer: KNativePointer) { super(pointer) } - static create1SrcDumper(node: AstNode | undefined, isDeclgen: boolean, isIsolatedDeclgen: boolean): SrcDumper { - return new SrcDumper(global.generatedEs2panda._CreateSrcDumper1(global.context, passNode(node), isDeclgen, isIsolatedDeclgen)) + static create1SrcDumper(node: AstNode | undefined, isDeclgen: boolean): SrcDumper { + return new SrcDumper(global.generatedEs2panda._CreateSrcDumper1(global.context, passNode(node), isDeclgen)) } /** @deprecated */ add(str: string): this { @@ -91,9 +91,6 @@ export class SrcDumper extends ArktsObject { get isDeclgen(): boolean { return global.generatedEs2panda._SrcDumperIsDeclgenConst(global.context, this.peer) } - get isIsolatedDeclgen(): boolean { - return global.generatedEs2panda._SrcDumperIsIsolatedDeclgenConst(global.context, this.peer) - } /** @deprecated */ dumpNode(key: string): this { global.generatedEs2panda._SrcDumperDumpNode(global.context, this.peer, key) -- Gitee From b79a03baa20b24102f1715793b2d114425fabe65 Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Thu, 19 Jun 2025 21:42:06 +0300 Subject: [PATCH 13/14] Fix some node api --- ui2abc/libarkts/src/arkts-api/peers/AstNode.ts | 1 - ui2abc/libarkts/src/arkts-api/utilities/public.ts | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/ui2abc/libarkts/src/arkts-api/peers/AstNode.ts b/ui2abc/libarkts/src/arkts-api/peers/AstNode.ts index 39d4542a1b..90453029e7 100644 --- a/ui2abc/libarkts/src/arkts-api/peers/AstNode.ts +++ b/ui2abc/libarkts/src/arkts-api/peers/AstNode.ts @@ -28,7 +28,6 @@ export abstract class AstNode extends ArktsObject { throwError(`attempted to create AstNode from nullptr`) } super(peer) - this.updateChildren() } public get originalPeer(): KNativePointer { diff --git a/ui2abc/libarkts/src/arkts-api/utilities/public.ts b/ui2abc/libarkts/src/arkts-api/utilities/public.ts index abd4c8a819..9b9d3b6dbf 100644 --- a/ui2abc/libarkts/src/arkts-api/utilities/public.ts +++ b/ui2abc/libarkts/src/arkts-api/utilities/public.ts @@ -19,6 +19,7 @@ import { KNativePointer, nullptr } from "@koalaui/interop" import { passNode, passNodeArray, unpackNodeArray, unpackNonNullableNode } from "./private" import { Es2pandaContextState, Es2pandaModifierFlags } from "../../generated/Es2pandaEnums" import type { AstNode } from "../peers/AstNode" +import { isSameNativeObject } from "../peers/ArktsObject" import { type AnnotationUsage, ClassDefinition, @@ -73,6 +74,9 @@ export function updateETSModuleByStatements( node: ETSModule, statements: readonly AstNode[], ): ETSModule { + if (isSameNativeObject(statements, node.statements)) { + return node + } global.generatedEs2panda._BlockStatementSetStatements(global.context, node.peer, passNodeArray(statements), statements.length) return node } -- Gitee From fd269d7aa7dc66a17e0020c47ebcc87b78bbb10f Mon Sep 17 00:00:00 2001 From: Igor Loginov Date: Thu, 19 Jun 2025 21:42:06 +0300 Subject: [PATCH 14/14] fix --- ui2abc/libarkts/native/src/bridges.cc | 2 +- ui2abc/libarkts/src/arkts-api/CompileWithCache.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ui2abc/libarkts/native/src/bridges.cc b/ui2abc/libarkts/native/src/bridges.cc index c57450bff3..a556ffa532 100644 --- a/ui2abc/libarkts/native/src/bridges.cc +++ b/ui2abc/libarkts/native/src/bridges.cc @@ -561,6 +561,6 @@ KInt impl_GenerateTsDeclarationsFromContext(KNativePointer contextPtr, KStringPt KBoolean exportAll, KBoolean isolated) { auto context = reinterpret_cast(contextPtr); - return GetImpl()->GenerateTsDeclarationsFromContext(context, outputDeclEts.data(), outputEts.data(), exportAll/*, isolated */); + return GetImpl()->GenerateTsDeclarationsFromContext(context, outputDeclEts.data(), outputEts.data(), exportAll, isolated); } KOALA_INTEROP_5(GenerateTsDeclarationsFromContext, KInt, KNativePointer, KStringPtr, KStringPtr, KBoolean, KBoolean) diff --git a/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts b/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts index 689ff9a7b0..56ce5997dc 100644 --- a/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts +++ b/ui2abc/libarkts/src/arkts-api/CompileWithCache.ts @@ -152,9 +152,11 @@ export function compileWithCache( } function cleanup() { - global.es2panda._DestroyContext( - context - ) + if (!it.output) { + global.es2panda._DestroyContext( + context + ) + } global.es2panda._DestroyConfig( config.peer ) -- Gitee