diff --git a/ets2panda/test/benchmarks/README.md b/ets2panda/test/benchmarks/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6cb9ab60eaaba61397d97c820e5181277b3c4eca --- /dev/null +++ b/ets2panda/test/benchmarks/README.md @@ -0,0 +1,21 @@ +# Logic +- Run es2panda for benchmark files from this directory, dump perf metrics to artifact dir +- Adding **time** and **mem** values of `#Parser and pipeline phases` and `#EmitProgram` for each runned benchmark +- Adding **time** and **mem** values of `#Parser and pipeline phases` and `#EmitProgram` in the corresponding files (`-max.txt`) +- Compare these values. Actual must be less than max for **time** and **mem**. + +# Ignore list +- Supports commets in `# ...` format +- Contains list of ignored tests (only basenames, e.g. `test.ets`). + +# Max values +Each file have companion: for `test.ets` companion is `test-max.txt`. This file contains max values for metrics. + +# Local reproduction +```bash +# static mode +bash runner.sh static /ets2panda/test/benchmarks + +# dynamic mode +bash runner.sh static /ets2panda/test/benchmarks +``` diff --git a/ets2panda/test/benchmarks/bench_1-max.txt b/ets2panda/test/benchmarks/bench_1-max.txt new file mode 100644 index 0000000000000000000000000000000000000000..009d8632846f37defb80ac15feedadcd9921e087 --- /dev/null +++ b/ets2panda/test/benchmarks/bench_1-max.txt @@ -0,0 +1,62 @@ +============================ es2panda perf metrics ============================= +==== Thread [23865] +#Parser and pipeline phases : time=8.14s mem=185MB +#phases/ConstantExpressionLowering : time=2.15s mem=264KB +#phases/TopLevelStatements : time=2.01s mem=114MB +#phases/CheckerPhase : time=934ms mem=21MB +#phases/ResolveIdentifiers : time=883ms mem=6MB +#EmitProgram : time=572ms mem=22MB +#phases/ScopesInitPhase : time=485ms mem=13MB +#GenerateProgram : time=424ms mem=0B +#phases/InterfaceObjectLiteralLowering : time=406ms mem=8MB +#phases/PartialExportClassGen : time=328ms mem=9MB +#phases/OptionalLowering : time=107ms mem=752KB +#phases/DefaultParametersInConstructorLowering : time=105ms mem=848KB +#phases/AmbientLowering : time=90.9ms mem=0B +#phases/PromiseVoidInferencePhase : time=88.6ms mem=0B +#phases/ExpressionLambdaConstruction : time=87.4ms mem=0B +#phases/InterfacePropertyDeclarationsPhase : time=86.2ms mem=1MB +#phases/Unbox : time=68.5ms mem=264KB +#phases/ResizableArrayConvert : time=62ms mem=500KB +#phases/InsertOptionalParametersAnnotation : time=60.8ms mem=1MB +#phases/EnumLoweringPhase : time=51.4ms mem=1MB +#phases/EnumPostCheckLoweringPhase : time=46.8ms mem=0B +#phases/DeclareOverloadLowering : time=42.5ms mem=0B +#phases/LambdaObjectConversion : time=5.63ms mem=324KB +#phases/CreateGenericBridges : time=2.59ms mem=260KB +#phases/BoxingForLocals : time=1.17ms mem=0B +#phases/UnionLowering : time=657us mem=0B +#phases/OptionalArgumentsLowering : time=610us mem=0B +#phases/ObjectIndexLowering : time=491us mem=0B +#phases/DefaultParametersLowering : time=478us mem=0B +#phases/BigIntLowering : time=473us mem=0B +#phases/LocalClassConstructionPhase : time=463us mem=0B +#phases/ObjectLiteralLowering : time=408us mem=0B +#phases/OpAssignmentLowering : time=378us mem=0B +#phases/CapturedVariables : time=363us mem=0B +#phases/LateInitializationConvert : time=322us mem=0B +#phases/RestTupleConstructionPhase : time=268us mem=0B +#phases/SetJumpTargetPhase : time=239us mem=0B +#phases/StringComparisonLowering : time=238us mem=0B +#phases/AsyncMethodLowering : time=222us mem=0B +#phases/RestArgsLowering : time=206us mem=0B +#phases/PrimitiveConversion : time=190us mem=0B +#phases/StringConstantsLowering : time=188us mem=0B +#phases/StringConstructorLowering : time=188us mem=0B +#phases/TypeFromLowering : time=186us mem=0B +#phases/SpreadConstructionPhase : time=182us mem=0B +#phases/ExtentionAccessorPhase : time=174us mem=0B +#phases/ArrayLiteralLowering : time=169us mem=0B +#phases/ExpandBracketsPhase : time=166us mem=0B +#phases/AnnotationCopyLowering : time=159us mem=0B +#phases/AnnotationCopyPostLowering : time=157us mem=0B +#phases/RecordLowering : time=150us mem=0B +#phases/ObjectIteratorLowering : time=147us mem=0B +#phases/ExportAnonymousConstPhase : time=24.5us mem=0B +#phases/plugins-after-parse : time=21.8us mem=0B +#phases/plugins-after-lowering : time=6.1us mem=0B +#phases/plugins-after-bind : time=5us mem=0B +#phases/CFGBuilder : time=4.6us mem=0B +#phases/plugins-after-check : time=4.2us mem=0B +#phases/PackageImplicitImport : time=2.5us mem=0B +#GenerateProgram/OptimizeBytecode : time=2.1us mem=0B diff --git a/ets2panda/test/benchmarks/bench_1.ets b/ets2panda/test/benchmarks/bench_1.ets new file mode 100644 index 0000000000000000000000000000000000000000..afa03ac57f549f5cdb54058f9f93373a76b6dfb7 --- /dev/null +++ b/ets2panda/test/benchmarks/bench_1.ets @@ -0,0 +1,343 @@ +/* + * Copyright (c) 2025 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +function test_create_empty_bigint(): void { + let a = new BigInt() + arktest.assertTrue(a.positive()) + arktest.assertEQ(a, 0n) + + let b = new BigInt() + arktest.assertTrue(b.positive()) + arktest.assertEQ(b, 0n) +} + +function test_invalid_bigint(): void { + // NOTE(kkonsw): implement validation +} + +function test_bigint_as_string(): void { + arktest.assertEQ(new BigInt("10").toString(), "10") + arktest.assertEQ(new BigInt("1000").toString(), "1000") + arktest.assertEQ(new BigInt("-1000").toString(), "-1000") + arktest.assertEQ(new BigInt("-1").toString(), "-1") + arktest.assertEQ(new BigInt("-10").toString(), "-10") + arktest.assertEQ(new BigInt("-100").toString(), "-100") + arktest.assertEQ(new BigInt("-100000000000000").toString(), "-100000000000000") + arktest.assertEQ(new BigInt("0").toString(), "0") +} + +function test_type(): void { + let num0 = 0n; + let num1 = 100_100_100_100_100_100n + let num2 = -57896044618658097711785492504343953926634992332820282019728792003956564819967n + + arktest.assertTrue(num0 instanceof bigint) + arktest.assertTrue(num1 instanceof bigint) + arktest.assertTrue(num2 instanceof bigint) +} + +function test_assignment(): void { + let a = -24059059045444224545405903904190343043049209420234290480n + let b = a + arktest.assertTrue(a instanceof bigint) + arktest.assertTrue(b instanceof bigint) + arktest.assertEQ(a, b) + + a = 123n; + arktest.assertTrue(a instanceof bigint) + arktest.assertEQ(a.toString(), "123") + arktest.assertEQ(a, 123n) + + const zero = 0n; + let c = zero; + arktest.assertTrue(zero instanceof bigint) + arktest.assertTrue(c instanceof bigint) + arktest.assertEQ(zero, c) +} + +function test_compare(): void { + const a = 24400569094091093912039019089543850580328542852805043n + const b = 34034240244909504590902901119302940942904944029040950n + + arktest.assertTrue(44493059209094029409209402940924902n < 140044940590459049067274048929058908989042385n) + arktest.assertTrue(44493059209094029409209402940924902n < a) + arktest.assertTrue(a < 34034240244909504590902901119302940942904944029040950n) + arktest.assertTrue(a < b) + + arktest.assertTrue(44493059209094029409209402940924902n <= 140044940590459049067274048929058908989042385n) + arktest.assertTrue(44493059209094029409209402940924902n <= a) + arktest.assertTrue(a <= 34034240244909504590902901119302940942904944029040950n) + arktest.assertTrue(a <= b) + + arktest.assertTrue(44493059209094029409209402940924902n <= 44493059209094029409209402940924902n) + arktest.assertTrue(24400569094091093912039019089543850580328542852805043n <= a) + arktest.assertTrue(a <= 24400569094091093912039019089543850580328542852805043n) + arktest.assertTrue(a <= a) + + arktest.assertTrue(40044940590459049067274048929058908989042385n > 44493059209094029409209402940924902n) + arktest.assertTrue(34034240244909504590902901119302940942904944029040950n > a) + arktest.assertTrue(a > 140044940590459049067274048929058908989042385n) + arktest.assertTrue(b > a) + + arktest.assertTrue(40044940590459049067274048929058908989042385n >= 44493059209094029409209402940924902n) + arktest.assertTrue(34034240244909504590902901119302940942904944029040950n >= a) + arktest.assertTrue(a >= 140044940590459049067274048929058908989042385n) + arktest.assertTrue(b >= a) + + arktest.assertTrue(44493059209094029409209402940924902n <= 44493059209094029409209402940924902n) + arktest.assertTrue(24400569094091093912039019089543850580328542852805043n <= a) + arktest.assertTrue(a <= 24400569094091093912039019089543850580328542852805043n) + arktest.assertTrue(a <= a) +} + +function test_literals() : void { + let num0 = 0n + arktest.assertEQ(num0.toString(), "0") + + let num1 = 127n + arktest.assertEQ(num1.toString(), "127") + + let num2 = 32767n + arktest.assertEQ(num2.toString(), "32767") + + let num3 = 2147483647n + arktest.assertEQ(num3.toString(), "2147483647") + + let num4 = 9223372036854775807n + arktest.assertEQ(num4.toString(), "9223372036854775807") + + let num5 = 170141183460469231731687303715884105727n + arktest.assertEQ(num5.toString(), "170141183460469231731687303715884105727") + + let num6 = 57896044618658097711785492504343953926634992332820282019728792003956564819967n + arktest.assertEQ(num6.toString(), "57896044618658097711785492504343953926634992332820282019728792003956564819967") + + let num1_n = -128n + arktest.assertEQ(num1_n.toString(), "-128") + + let num2_n = -32768n + arktest.assertEQ(num2_n.toString(), "-32768") + + let num3_n = -2147483648n + arktest.assertEQ(num3_n.toString(), "-2147483648") + + let num4_n = -9223372036854775808n + arktest.assertEQ(num4_n.toString(), "-9223372036854775808") + + let num5_n = -170141183460469231731687303715884105728n + arktest.assertEQ(num5_n.toString(), "-170141183460469231731687303715884105728") + + let num6_n = -57896044618658097711785492504343953926634992332820282019728792003956564819968n + arktest.assertEQ(num6_n.toString(), "-57896044618658097711785492504343953926634992332820282019728792003956564819968") + + let num1_sep = 1_991_653_125_841_217_555_434419_9091_123000000_3_3313_5775_3282_29n + arktest.assertEQ(num1_sep.toString(), "19916531258412175554344199091123000000333135775328229") + + let num2_sep = -422_12_3333_9844_3333_3443_34111_43434_1111_11_1_3_3_411909_990081n + arktest.assertEQ(num2_sep.toString(), "-4221233339844333334433411143434111111133411909990081") + + let num0_t: bigint = 0n + arktest.assertEQ(num0_t.toString(), "0") + + let num1_t: bigint = 57896044618658097711785492504343953926634992332820282019728792003956564819967n + arktest.assertEQ(num1_t.toString(), "57896044618658097711785492504343953926634992332820282019728792003956564819967") + + let num2_t: bigint = -9223372036854775808n + arktest.assertEQ(num2_t.toString(), "-9223372036854775808") + + let num3_t: bigint = 1_991_653_125_841_217_555_434419_9091_123000000_3_3313_5775_3282_29n + arktest.assertEQ(num3_t.toString(), "19916531258412175554344199091123000000333135775328229") + + let num4_t: bigint = -422_12_3333_9844_3333_3443_34111_43434_1111_11_1_3_3_411909_990081n + arktest.assertEQ(num4_t.toString(), "-4221233339844333334433411143434111111133411909990081") + + const num0_c = 0n + arktest.assertEQ(num0_c.toString(), "0") + + const num1_c = 1267650600228229401496703205376n + arktest.assertEQ(num1_c.toString(), "1267650600228229401496703205376") + + const num2_c = -1427247692705959881058285969449495136382746624n + arktest.assertEQ(num2_c.toString(), "-1427247692705959881058285969449495136382746624") + + const num3_c = 4_000_000_000_000_000_000_000_100n + arktest.assertEQ(num3_c.toString(), "4000000000000000000000100") + + const num4_c: bigint = -7777777_666666_55555_4444_333_22_1n + arktest.assertEQ(num4_c.toString(), "-7777777666666555554444333221") +} + +function test_cast(): void { + const v = 1559053 + const b: byte = 44 + const s: short = -17600 + const i: int = 1150483640 + const l: long = -8223372036854775808 + + // NOTE(kkonsw): casts currently do not work +} + +function test_bigint_methods(): void { + const b: byte = 44 + const s: short = -17600 + const i: int = 1150483640 + const l: long = -8223372036854775808 + + /* Testing BigInt constructor */ + let n0 = new BigInt(0) + arktest.assertEQ(n0, 0n) + arktest.assertEQ(n0.toString(), "0") + + let n1 = new BigInt(654093) + arktest.assertEQ(n1.toString(), "654093") + arktest.assertEQ(n1, 654093n) + + let n2 = new BigInt(b) + arktest.assertEQ(n2.toString(), "44") + arktest.assertEQ(n2, 44n) + + let n3 = new BigInt(s) + arktest.assertEQ(n3.toString(), "-17600") + arktest.assertEQ(n3, -17600n) + + let n4 = new BigInt(i) + arktest.assertEQ(n4.toString(), "1150483640") + arktest.assertEQ(n4, 1150483640n) + + let n5 = new BigInt(l) + arktest.assertEQ(n5.toString(), "-8223372036854775808") + arktest.assertEQ(n5, -8223372036854775808n) + + let dec = new BigInt("-12392320390239294724747283477947923471101032") + arktest.assertEQ(dec, -12392320390239294724747283477947923471101032n) + + const n7 = 12392320390239294724747283477947923471101032n + + /* Testing asIntN() static method */ + arktest.assertEQ(BigInt.asIntN(0, n7), 0n) + arktest.assertEQ(BigInt.asIntN(8, n7), 104n) + arktest.assertEQ(BigInt.asIntN(16, n7), 27752n) + arktest.assertEQ(BigInt.asIntN(32, n7), -737317784n) + arktest.assertEQ(BigInt.asIntN(64, n7), -7098331616643290008n) + + /* Testing asUintN() static method */ + arktest.assertEQ(BigInt.asUintN(0, n7), 0n) + arktest.assertEQ(BigInt.asUintN(8, n7), 104n) + arktest.assertEQ(BigInt.asUintN(16, n7), 27752n) + arktest.assertEQ(BigInt.asUintN(32, n7), 3557649512n) + arktest.assertEQ(BigInt.asUintN(64, n7), 11348412457066261608n) +} + +function test_shift(): void { + const a = 245599210405555256299145n + /* Testing left shift (<<) */ + arktest.assertEQ(a << 100n, 311333986486181324779687697000809288883015536628203520n) + arktest.assertEQ(a << 0n, a) + + /* Testing right shift (>>) */ + arktest.assertEQ(a >> 60n, 213023n) + arktest.assertEQ(a >> 0n, a) +} + +function test_scientific(): void { + arktest.assertEQ(new BigInt(0.0e0).toString(), "0", "BigInt(0.0e0)") + arktest.assertEQ(new BigInt(0.0e+0).toString(), "0", "BigInt(0.0e+0)") + arktest.assertEQ(new BigInt(0.0e-0).toString(), "0", "BigInt(0.0e-0).toString()") + arktest.assertEQ(new BigInt(-0.0e0).toString(), "0", "BigInt(-0.0e0).toString()") + arktest.assertEQ(new BigInt(1e23).toString(), "100000000000000000000000", "BigInt(1e22)") + arktest.assertEQ(new BigInt(1e+23).toString(), "100000000000000000000000", "BigInt(1e+22)") + arktest.assertEQ(new BigInt(-1e23).toString(), "-100000000000000000000000", "BigInt(-1e22)") + arktest.assertEQ(new BigInt(-1e+23).toString(), "-100000000000000000000000", "BigInt(-1e+22)") + + arktest.assertEQ(new BigInt(1.234567e10).toString(), "12345670000", "BigInt(1.234567e10).toString()") + arktest.assertEQ(new BigInt(1.234567e20).toString(), "123456700000000000000", "BigInt(1.234567e20).toString()") + + arktest.assertEQ(new BigInt(1.2345678912e21).toString(), "1234567891200000000000", "BigInt(1.2345678912e21)") + arktest.assertEQ(new BigInt(1.2345678912e+21).toString(), "1234567891200000000000", "BigInt(1.2345678912e+21)") + arktest.assertEQ(new BigInt(-1.2345678912e21).toString(), "-1234567891200000000000", "BigInt(-1.2345678912e21)") +} + +function test_double(): void { + let bigIntFromDouble = (x: number): boolean => { + try { + let b = new BigInt(x); + } catch (e) { + return e instanceof Error; + } + return false + }; + + arktest.assertTrue(bigIntFromDouble(0.1), "BigInt(0.1)") + arktest.assertTrue(bigIntFromDouble(-0.1), "BigInt(-0.1)") + arktest.assertTrue(bigIntFromDouble(42.1234567), "BigInt(42.1234567)") + arktest.assertTrue(bigIntFromDouble(1.234567e2), "BigInt(1.234567e2)") + arktest.assertTrue(bigIntFromDouble(1e-22), "BigInt(1e-22)") + arktest.assertTrue(bigIntFromDouble(-1e-22), "BigInt(-1e-22)") + arktest.assertTrue(bigIntFromDouble(1.88e-20), "BigInt(1.88e-20)") + arktest.assertTrue(bigIntFromDouble(1.2345678848e-21), "BigInt(1.2345678484e-21)") +} + +function test_sign(): void { + arktest.assertEQ(new BigInt().negative() , false , 'BigInt() is negative') + arktest.assertEQ(new BigInt(0).negative() , false , 'BigInt(0) is negative') + arktest.assertEQ(new BigInt(-0).negative() , false , 'BigInt(-0) is negative') + arktest.assertEQ(new BigInt(0.0).negative() , false , 'BigInt(0.0) is negative') + arktest.assertEQ(new BigInt(-0.0).negative(), false , 'BigInt(-0.0) is negative') + arktest.assertEQ(new BigInt("0").negative() , false , 'BigInt("0") is negative') + arktest.assertEQ(new BigInt("-0").negative(), false , 'BigInt("-0") is negative') + + arktest.assertEQ(new BigInt().positive() , true , 'BigInt() is not positive') + arktest.assertEQ(new BigInt(0).positive() , true , 'BigInt(0) is not positive') + arktest.assertEQ(new BigInt(-0).positive() , true , 'BigInt(-0) is not positive') + arktest.assertEQ(new BigInt(0.0).positive() , true , 'BigInt(0.0) is not positive') + arktest.assertEQ(new BigInt(-0.0).positive(), true , 'BigInt(-0.0) is not positive') + arktest.assertEQ(new BigInt("0").positive() , true , 'BigInt("0") is not positive') + arktest.assertEQ(new BigInt("-0").positive(), true , 'BigInt("-0") is not positive') + + arktest.assertEQ(new BigInt("0").negate().positive() , true , 'BigInt("0").negate() is not positive') + arktest.assertEQ(new BigInt("-0").negate().positive(), true , 'BigInt("-0").negate() is not positive') + + arktest.assertTrue((new BigInt("1") + new BigInt("-1")).positive() , 'BigInt(1 + -1) is not positive') + arktest.assertTrue((new BigInt("-1") + new BigInt("1")).positive() , 'BigInt(-1 + 1) is not positive') + arktest.assertTrue((new BigInt("1") - new BigInt("1")).positive() , 'BigInt(1 - 1) is not positive') + arktest.assertTrue((new BigInt("-1") - new BigInt("-1")).positive() , 'BigInt(-1 - -1) is not positive') + arktest.assertTrue((new BigInt("1") * new BigInt("0")).positive() , 'BigInt(1 * 0) is not positive') + arktest.assertTrue((new BigInt("-1") * new BigInt("0")).positive() , 'BigInt(-1 * 0) is not positive') + arktest.assertTrue((new BigInt("0") * new BigInt("1")).positive() , 'BigInt(0 * 1) is not positive') + arktest.assertTrue((new BigInt("0") * new BigInt("-1")).positive() , 'BigInt(0 * -1) is not positive') + arktest.assertTrue((new BigInt("0") / new BigInt("1")).positive() , 'BigInt(0 / 1) is not positive') + arktest.assertTrue((new BigInt("0") / new BigInt("-1")).positive() , 'BigInt(0 / -1) is not positive') + arktest.assertTrue((new BigInt("4") % new BigInt("2")).positive() , 'BigInt(4 % 2) is not positive') + arktest.assertTrue((new BigInt("4") % new BigInt("-2")).positive() , 'BigInt(4 % -2) is not positive') + arktest.assertTrue((new BigInt("-4") % new BigInt("2")).positive() , 'BigInt(-4 % 2) is not positive') + arktest.assertTrue((new BigInt("-4") % new BigInt("-2")).positive() , 'BigInt(-4 % -2) is not positive') +} + +function main() : void { + test_create_empty_bigint(); + test_bigint_as_string(); + test_invalid_bigint(); + test_type(); + test_assignment(); + test_compare(); + test_literals(); + test_cast(); + test_bigint_methods(); + test_shift(); + test_double(); + test_sign(); + // NOTE(aakmaev): Enable after fix #17683. + // test_scientific(); +} diff --git a/ets2panda/test/benchmarks/etsstdlib-max.txt b/ets2panda/test/benchmarks/etsstdlib-max.txt new file mode 100644 index 0000000000000000000000000000000000000000..a88a98346a7a890f4a3ec3a538764af9c61da9bb --- /dev/null +++ b/ets2panda/test/benchmarks/etsstdlib-max.txt @@ -0,0 +1,62 @@ +============================ es2panda perf metrics ============================= +==== Thread [23643] +#Parser and pipeline phases : time=21.8s mem=426MB +#phases/CheckerPhase : time=5.62s mem=75MB +#EmitProgram : time=4.61s mem=124MB +#phases/TopLevelStatements : time=2.24s mem=115MB +#phases/ConstantExpressionLowering : time=2.11s mem=320KB +#GenerateProgram : time=1.91s mem=0B +#phases/Unbox : time=1.89s mem=41MB +#phases/LambdaObjectConversion : time=1.75s mem=23MB +#phases/CreateGenericBridges : time=1.1s mem=85MB +#phases/OpAssignmentLowering : time=1.06s mem=18MB +#phases/ResolveIdentifiers : time=939ms mem=6MB +#phases/ScopesInitPhase : time=470ms mem=13MB +#phases/InterfaceObjectLiteralLowering : time=439ms mem=8MB +#phases/PartialExportClassGen : time=348ms mem=9MB +#phases/BoxingForLocals : time=321ms mem=2MB +#phases/ObjectIndexLowering : time=317ms mem=2MB +#phases/PrimitiveConversion : time=308ms mem=4MB +#phases/RestArgsLowering : time=288ms mem=5MB +#phases/ArrayLiteralLowering : time=219ms mem=4MB +#phases/UnionLowering : time=146ms mem=0B +#phases/ObjectIteratorLowering : time=128ms mem=1MB +#phases/ObjectLiteralLowering : time=126ms mem=236KB +#phases/OptionalArgumentsLowering : time=124ms mem=260KB +#phases/LocalClassConstructionPhase : time=120ms mem=0B +#phases/OptionalLowering : time=104ms mem=752KB +#phases/DefaultParametersInConstructorLowering : time=102ms mem=528KB +#phases/DefaultParametersLowering : time=98.8ms mem=264KB +#phases/LateInitializationConvert : time=90.5ms mem=0B +#phases/AmbientLowering : time=88.1ms mem=0B +#phases/PromiseVoidInferencePhase : time=87.1ms mem=0B +#phases/ExpressionLambdaConstruction : time=84.9ms mem=0B +#phases/InterfacePropertyDeclarationsPhase : time=79.3ms mem=1MB +#phases/StringComparisonLowering : time=68.8ms mem=0B +#phases/InsertOptionalParametersAnnotation : time=59.9ms mem=1MB +#phases/StringConstructorLowering : time=59.2ms mem=0B +#phases/ResizableArrayConvert : time=57.4ms mem=496KB +#phases/AsyncMethodLowering : time=56.8ms mem=0B +#phases/TypeFromLowering : time=56.1ms mem=0B +#phases/EnumPostCheckLoweringPhase : time=55.2ms mem=204KB +#phases/CapturedVariables : time=54.1ms mem=0B +#phases/ExpandBracketsPhase : time=53.5ms mem=0B +#phases/RecordLowering : time=52.6ms mem=0B +#phases/EnumLoweringPhase : time=51.4ms mem=1MB +#phases/ExtentionAccessorPhase : time=50.8ms mem=0B +#phases/SetJumpTargetPhase : time=50.3ms mem=0B +#phases/BigIntLowering : time=46.9ms mem=264KB +#phases/RestTupleConstructionPhase : time=44.4ms mem=264KB +#phases/DeclareOverloadLowering : time=43.1ms mem=0B +#phases/AnnotationCopyPostLowering : time=40.1ms mem=0B +#phases/SpreadConstructionPhase : time=39ms mem=0B +#phases/AnnotationCopyLowering : time=38.8ms mem=0B +#phases/plugins-after-parse : time=18.7us mem=0B +#phases/StringConstantsLowering : time=12.3us mem=0B +#phases/ExportAnonymousConstPhase : time=10.4us mem=0B +#phases/CFGBuilder : time=6.2us mem=0B +#phases/plugins-after-bind : time=6.1us mem=0B +#phases/plugins-after-lowering : time=6us mem=0B +#phases/plugins-after-check : time=4.9us mem=0B +#GenerateProgram/OptimizeBytecode : time=3us mem=0B +#phases/PackageImplicitImport : time=2.2us mem=0B diff --git a/ets2panda/test/benchmarks/ignored_list.txt b/ets2panda/test/benchmarks/ignored_list.txt new file mode 100644 index 0000000000000000000000000000000000000000..b9fe93c3f5c5f4d69640252e3ebaf968968309a4 --- /dev/null +++ b/ets2panda/test/benchmarks/ignored_list.txt @@ -0,0 +1,2 @@ +# Comment example +# test.ets diff --git a/ets2panda/test/benchmarks/runner/benchmark_tools.sh b/ets2panda/test/benchmarks/runner/benchmark_tools.sh new file mode 100644 index 0000000000000000000000000000000000000000..9c6b4ffd2e4925da7e267c2149af9b64133fa8a1 --- /dev/null +++ b/ets2panda/test/benchmarks/runner/benchmark_tools.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +compare_results() { + local benchmark_name="$1" + local max_file="$2" + local actual_file="$3" + local regression_gap=${4:-0} + + local max_time_base="$(get_total_time_from_file $max_file)" + local max_mem_base="$(get_total_mem_from_file $max_file)" + local actual_time="$(get_total_time_from_file $actual_file)" + local actual_mem="$(get_total_mem_from_file $actual_file)" + + local max_time=$(awk -v base="$max_time_base" -v gap="$regression_gap" 'BEGIN { printf "%.0f", base * (1 + gap) }') + local max_mem=$(awk -v base="$max_mem_base" -v gap="$regression_gap" 'BEGIN { printf "%.0f", base * (1 + gap) }') + + if (( actual_time > max_time || actual_mem > max_mem )); then + echo "FAILED: Benchmark failed: '$benchmark_name'" + if (( actual_time > max_time )); then + echo " Time limit exceeded: actual is ${actual_time}ns, limit (incl. gap) is ${max_time}ns (base: ${max_time_base}ns)" + fi + if (( actual_mem > max_mem )); then + echo " Memory limit exceeded: actual is ${actual_mem}B, limit (incl. gap) is ${max_mem}B (base: ${max_mem_base}B)" + fi + echo + ((++FAILED_COUNT)) + else + echo "[Info]: Passed benchmark for '$benchmark_name'." + ((++PASSED_COUNT)) + fi +} + +run_static_benchmarks_from_dir() { + local test_dir="$1" + local artifacts_dir="$2" + local es2panda="$3" + + mkdir -p "$artifacts_dir" || error_exit "Can't create artifacts_dir: '$artifacts_dir'" + + local tests="$(collect_tests_from_dir $test_dir)" + for test in ${tests[@]}; do + local base_name=$(basename "$test" .ets) + local expected_file="$(dirname "$test")/$base_name-max.txt" + local output_abc_file="$artifacts_dir/${base_name}.abc" + local output_log_file="$artifacts_dir/${base_name}.log" + + $es2panda --dump-perf-metrics --output=$output_abc_file $test > $output_log_file || \ + (cat $output_log_file && error_exit "Run es2panda for file '$test' failed.") + + compare_results "$test (static)" $expected_file $output_log_file $REGRESSION_GAP + done +} + +run_static_benchmarks_stdlib() { + local expected_file="$1" + local artifacts_dir="$2" + local es2panda="$3" + + mkdir -p "$artifacts_dir" || error_exit "Can't create artifacts_dir: '$artifacts_dir'" + + local output_abc_file="$artifacts_dir/etsstdlib.abc" + local output_log_file="$artifacts_dir/etsstdlib.log" + + $es2panda --dump-perf-metrics --extension=ets --output=$output_abc_file --gen-stdlib=true > $output_log_file || \ + (cat $output_log_file && error_exit "Run es2panda for file 'stdlib' failed.") + + compare_results "stdlib (static)" $expected_file $output_log_file $REGRESSION_GAP +} + +compare_current_and_master() { + local test_dir="$1" + local artifacts_dir="$2" + + local current="$artifacts_dir/current/etsstdlib.log" + local master="$artifacts_dir/master/etsstdlib.log" + compare_results "stdlib (dynamic)" $master $current $REGRESSION_GAP + + local tests="$(collect_tests_from_dir $test_dir)" + for test in ${tests[@]}; do + local base_name=$(basename "$test" .ets) + local current="$artifacts_dir/current/${base_name}.log" + local master="$artifacts_dir/master/${base_name}.log" + + compare_results "$test (dynamic)" $master $current $REGRESSION_GAP + done +} diff --git a/ets2panda/test/benchmarks/runner/parse_reports_tools.sh b/ets2panda/test/benchmarks/runner/parse_reports_tools.sh new file mode 100644 index 0000000000000000000000000000000000000000..6286597bbeb99c23a277cc9550a2fb301eb7421d --- /dev/null +++ b/ets2panda/test/benchmarks/runner/parse_reports_tools.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +get_time_ns() { + local filename="$1" + local pattern="$2" + + grep "$pattern" "$filename" | awk ' + BEGIN { + mult["s"] = 1000000000 + mult["ms"] = 1000000 + mult["us"] = 1000 + mult["ns"] = 1 + } + { + sub(/.*time=/, "") + value_time = $0 + unit_time = $0 + sub(/[a-z].*/, "", value_time) + sub(/[0-9.]+/, "", unit_time) + sub(/ .*/, "", unit_time) + printf "%.0f\n", value_time * mult[unit_time] + } + ' +} + +get_total_time_from_file() { + local filename="$1" + [[ -f "$filename" ]] || error_exit "Can't find file with perf stat: '$filename'" + + local time1_ns=$(get_time_ns "$filename" "^#Parser and pipeline phases") + local time2_ns=$(get_time_ns "$filename" "^#EmitProgram") + + echo $(( ${time1_ns:-0} + ${time2_ns:-0} )) +} + +get_mem_b() { + local filename="$1" + local pattern="$2" + + grep "$pattern" "$filename" | awk ' + BEGIN { + mult["GB"] = 1024 * 1024 * 1024 + mult["MB"] = 1024 * 1024 + mult["KB"] = 1024 + mult["B"] = 1 + } + { + sub(/\r$/, "") + sub(/.*mem=/, "") + value_mem = $0 + unit_mem = $0 + sub(/[A-Za-z].*/, "", value_mem) + sub(/[0-9.]+/, "", unit_mem) + sub(/ .*/, "", unit_mem) + printf "%.0f\n", value_mem * mult[unit_mem] + } + ' +} + +get_total_mem_from_file() { + local filename="$1" + [[ -f "$filename" ]] || error_exit "Can't find file with perf stat: '$filename'" + + local mem1_b=$(get_mem_b "$filename" "^#Parser and pipeline phases") + local mem2_b=$(get_mem_b "$filename" "^#EmitProgram") + echo $(( ${mem1_b:-0} + ${mem2_b:-0} )) +} diff --git a/ets2panda/test/benchmarks/runner/runner.sh b/ets2panda/test/benchmarks/runner/runner.sh new file mode 100755 index 0000000000000000000000000000000000000000..23c7a833698e966e3bd55e4b3dee2a83d46e34e4 --- /dev/null +++ b/ets2panda/test/benchmarks/runner/runner.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +RUNNER_ROOT_DIR="$(dirname "$0")" +source "$RUNNER_ROOT_DIR/parse_reports_tools.sh" +source "$RUNNER_ROOT_DIR/benchmark_tools.sh" +source "$RUNNER_ROOT_DIR/test_lists_tools.sh" + +error_exit() { + echo "ERROR: $1" >&2 + exit 1 +} + +check_params() { + [[ "$MODE" == "static" || "$MODE" == "dynamic" ]] || error_exit "Only 'static' or 'dynamic' mode supported." + [[ -f "$ES2PANDA" ]] || error_exit "es2panda not found at: '$ES2PANDA'" + [[ -d "$TESTS_DIR" ]] || error_exit "Benchmark test directory not found at: '$TESTS_DIR'" + + if [[ "$MODE" == "dynamic" ]]; then + [[ -f "$ES2PANDA_MASTER" ]] || error_exit "es2panda for master not found at: '$ES2PANDA_MASTER'" + + if ! echo "$REGRESSION_GAP" | awk '{ exit !($1 >= 0 && $1 <= 1) }'; then + error_exit "REGRESSION_GAP must be a number between 0 and 1, got: '$REGRESSION_GAP'" + fi + fi +} + +# Supports pass arguments via env +MODE="${MODE:-$1}" +ES2PANDA="${ES2PANDA:-$2}" +TESTS_DIR="${TESTS_DIR:-$3}" +ARTIFACTS_DIR="${ARTIFACTS_DIR:-$4}" + +# Only for dynamic mode: +ES2PANDA_MASTER="${ES2PANDA_MASTER:-$5}" +REGRESSION_GAP=0.05 # Acceptable degradation value (` actual <= master * (1 + REGRESSION_GAP) `) + + +# TODO: remove this debug output +echo "DEBUG OUTPUT OF VARIABLES:" +echo "MODE='$MODE'" +echo "ES2PANDA='$ES2PANDA'" +echo "TESTS_DIR='$TESTS_DIR'" +echo "ARTIFACTS_DIR='$ARTIFACTS_DIR'" +echo "ES2PANDA_MASTER='$ES2PANDA_MASTER'" +echo + +check_params + +PASSED_COUNT=0 +FAILED_COUNT=0 + +run_static_benchmarks_stdlib "$TESTS_DIR/etsstdlib-max.txt" "$ARTIFACTS_DIR/current" "$ES2PANDA" +run_static_benchmarks_from_dir "$TESTS_DIR" "$ARTIFACTS_DIR/current" "$ES2PANDA" + +if [[ "$MODE" == "dynamic" ]]; then + run_static_benchmarks_stdlib "$TESTS_DIR/etsstdlib-max.txt" "$ARTIFACTS_DIR/master" "$ES2PANDA_MASTER" + run_static_benchmarks_from_dir "$TESTS_DIR" "$ARTIFACTS_DIR/master" "$ES2PANDA_MASTER" + compare_current_and_master "$TESTS_DIR" "$ARTIFACTS_DIR" +fi + +echo -e """ +===================== + SUMMARY +Passed: $PASSED_COUNT +Failed: $FAILED_COUNT +Total: $((PASSED_COUNT + FAILED_COUNT)) +""" + +exit $FAILED_COUNT diff --git a/ets2panda/test/benchmarks/runner/test_lists_tools.sh b/ets2panda/test/benchmarks/runner/test_lists_tools.sh new file mode 100644 index 0000000000000000000000000000000000000000..e45b4d62ed482665a050399d50b263ddbf376e7b --- /dev/null +++ b/ets2panda/test/benchmarks/runner/test_lists_tools.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright (c) 2025 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -e + +clear_tests_list() { + local file="$1" + grep -v '^$\|^#' "$file" | sed 's#^#/#g' +} + +collect_tests_from_dir() { + local test_dir="$1" + local ignored_list="$test_dir/ignored_list.txt" + [[ -d "$test_dir" ]] || error_exit "Can't find test directory: '$test_dir'" + + local tests="$(find "$test_dir" -type f -name "*.ets" -print)" + if [[ -f "$ignored_list" ]]; then + tests="$(echo -e "$tests" | grep -v -F -f <(clear_tests_list "$ignored_list"))" + else + echo "[Warning]: Not found ignore list: '$ignored_list'" + fi + + echo -e "$tests" +}