From 9d3bb64b591e2cf42d06d985bc715cef2d2851d3 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 14:29:45 +0300 Subject: [PATCH 01/36] The first ETS test running structure --- test/runner/runner_ets.py | 37 +++++++++++++++++++++++++++++++++++ test/runner/runner_ets_cts.py | 12 ++++++++++++ test/runner/test_ets.py | 17 ++++++++++++++++ test/runner/test_ets_cts.py | 13 ++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 test/runner/runner_ets.py create mode 100644 test/runner/runner_ets_cts.py create mode 100644 test/runner/test_ets.py create mode 100644 test/runner/test_ets_cts.py diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py new file mode 100644 index 000000000..de3b5bc1c --- /dev/null +++ b/test/runner/runner_ets.py @@ -0,0 +1,37 @@ +import subprocess +import sys +from datetime import datetime +from glob import glob +from os import path, environ, makedirs +from typing import List + +from configuration_kind import ConfigurationKind +from fail_kind import FailKind +from params import TestEnv +from report_format import ReportFormat +from runner_base import Runner +from utils import write_2_file + +INDEX_TITLE = "${Title}" +INDEX_OPTIONS = "${Options}" +INDEX_TOTAL = "${Total}" +INDEX_PASSED = "${Passed}" +INDEX_FAILED = "${Failed}" +INDEX_IGNORED = "${Ignored}" +INDEX_EXCLUDED_LISTS = "${ExcludedThroughLists}" +INDEX_EXCLUDED_OTHER = "${ExcludedByOtherReasons}" +INDEX_TEST_NAME = "${TestName}" +INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" + + +class RunnerETS(Runner): + def __init__(self, args, name): + Runner.__init__(self, args, name) + self.cmd_env = environ.copy() + + def _generate_ets_stdlib(): + pass + + + + \ No newline at end of file diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py new file mode 100644 index 000000000..daa27ad29 --- /dev/null +++ b/test/runner/runner_ets_cts.py @@ -0,0 +1,12 @@ +from runner_base import correct_path, get_test_id +from runner_ets import RunnerETS +from test_ets_cts import TestJSHermes +from util_hermes import UtilHermes + + +class RunnerETSCTS(RunnerJS): + def __init__(self, args): + RunnerETS.__init__(self, args, "hermes") + + + \ No newline at end of file diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py new file mode 100644 index 000000000..8ec62c3d5 --- /dev/null +++ b/test/runner/test_ets.py @@ -0,0 +1,17 @@ +import subprocess +import sys +from os import path, remove +from typing import List + +from configuration_kind import ConfigurationKind +from fail_kind import FailKind +from params import Params, TestReport +from test_base import Test + + +class TestETS(Test): + def __init__(self, test_env, test_path, flags, test_id, update_expected): + Test.__init__(self, test_env, test_path, flags, test_id, update_expected) + # If test fails it contains reason (of FailKind enum) of first failed step + # It's supposed if the first step is failed then no step is executed further + self.fail_kind = None \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py new file mode 100644 index 000000000..80e421fa8 --- /dev/null +++ b/test/runner/test_ets_cts.py @@ -0,0 +1,13 @@ +from os import path, makedirs + +from configuration_kind import ConfigurationKind +from test_ets import TestETS +from utils import purify + +class TestETS_CTS(TestETS): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) + self.tmp_dir = path.join(path.sep, "tmp", "cts") + makedirs(self.tmp_dir, exist_ok=True) + self.util = self.test_env.util + -- Gitee From eff1d656c0ad345455016f55266b4551b86beead Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 14:29:51 +0300 Subject: [PATCH 02/36] add etc cts specific runner --- test/runner/runner_ets_cts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index daa27ad29..db57261d1 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -4,9 +4,13 @@ from test_ets_cts import TestJSHermes from util_hermes import UtilHermes -class RunnerETSCTS(RunnerJS): +class RunnerETSCTS(RunnerETS): def __init__(self, args): - RunnerETS.__init__(self, args, "hermes") + RunnerETS.__init__(self, args, "ets-cts") + + def _generate_ets_code_from_template(): + pass + \ No newline at end of file -- Gitee From a28a5fecaf460c0b6daffb779cbcc3c3db7fffe8 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 18:32:28 +0300 Subject: [PATCH 03/36] add stdlib structure for runner --- test/runner/runner.py | 8 ++++++++ test/runner/runner_ets_cts.py | 5 ++--- test/runner/runner_ets_stdlib.py | 15 +++++++++++++++ test/runner/starter.py | 8 ++++++-- test/runner/test_ets_stdlib.py | 13 +++++++++++++ 5 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 test/runner/runner_ets_stdlib.py create mode 100644 test/runner/test_ets_stdlib.py diff --git a/test/runner/runner.py b/test/runner/runner.py index 607e8cb25..74ca8960a 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -4,6 +4,8 @@ from dotenv import load_dotenv from runner_js_hermes import RunnerJSHermes from runner_js_parser import RunnerJSParser from runner_js_test262 import RunnerJSTest262 +from runner_ets_cts import RunnerETS_CTS +from runner_ets_stdlib import RunnerETS_STDLIB from starter import get_args @@ -27,6 +29,12 @@ def main(): if args.hermes: runners.append(RunnerJSHermes(args)) + if args.ets_cts: + runners.append(RunnerETS_CTS(args)) + + if args.ets_stdlib: + runners.append(RunnerETS_STDLIB(args)) + failed_tests = 0 for runner in runners: diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index db57261d1..55b0dafbe 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -1,10 +1,9 @@ from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_cts import TestJSHermes -from util_hermes import UtilHermes +from test_ets_cts import TestETS_CTS -class RunnerETSCTS(RunnerETS): +class RunnerETS_CTS(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-cts") diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py new file mode 100644 index 000000000..cf801d317 --- /dev/null +++ b/test/runner/runner_ets_stdlib.py @@ -0,0 +1,15 @@ +from runner_base import correct_path, get_test_id +from runner_ets import RunnerETS +from test_ets_stdlib import TestETS_STDLIB + + +class RunnerETS_STDLIB(RunnerETS): + def __init__(self, args): + RunnerETS.__init__(self, args, "ets-cts") + + def _generate_ets_code_from_template(): + pass + + + + \ No newline at end of file diff --git a/test/runner/starter.py b/test/runner/starter.py index a1f1792f1..553faa5f3 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -21,7 +21,8 @@ def is_file(parser, arg): def check_timeout(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError(f"{value} is an invalid timeout value") + raise argparse.ArgumentTypeError( + f"{value} is an invalid timeout value") return ivalue @@ -43,7 +44,10 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - + parser.add_argument('--ets-cts', action='store_true', dest='ets_sts', + default=False, help='run test agaisnt ETS specification') + parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', + default=False, help='run test agaisnt ETS specification') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), help='directory with test file. If not set the module directory is used') diff --git a/test/runner/test_ets_stdlib.py b/test/runner/test_ets_stdlib.py new file mode 100644 index 000000000..dd1b3fb07 --- /dev/null +++ b/test/runner/test_ets_stdlib.py @@ -0,0 +1,13 @@ +from os import path, makedirs + +from configuration_kind import ConfigurationKind +from test_ets import TestETS +from utils import purify + +class TestETS_STDLIB(TestETS): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) + self.tmp_dir = path.join(path.sep, "tmp", "cts") + makedirs(self.tmp_dir, exist_ok=True) + self.util = self.test_env.util + -- Gitee From e6f4068e23827302b0c233c26696f783987ad303 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 18 Oct 2022 18:16:59 +0300 Subject: [PATCH 04/36] add stdlib generating, cts code generating --- test/runner/runner_ets.py | 47 +++++++++++++++++++++++++++++++---- test/runner/runner_ets_cts.py | 32 ++++++++++++++++++++++-- test/runner/starter.py | 2 +- test/runner/test_ets.py | 12 +++++++-- test/runner/test_ets_cts.py | 4 +-- 5 files changed, 85 insertions(+), 12 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index de3b5bc1c..0f028d6e3 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -4,6 +4,9 @@ from datetime import datetime from glob import glob from os import path, environ, makedirs from typing import List +from pathlib import Path +import shlex + from configuration_kind import ConfigurationKind from fail_kind import FailKind @@ -25,13 +28,47 @@ INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" class RunnerETS(Runner): - def __init__(self, args, name): + def __init__(self, args, name:str): Runner.__init__(self, args, name) + output_path = path.join(self.stdlib_path, self.stdlib_name) + self.cmd_env = environ.copy() + self.test_root=args.test_root + self.compiler=path.join(self.build_dir, "bin", "es2panda") + self.runtime = path.join(self.build_dir, "bin", "ark") + self.runtime_args = f"--boot-panda={output_path} --load_runtime=ets" self.cmd_env = environ.copy() + self.stdlib_name="etsstdlib.abc" + self.bytecode_path=path.join("/tmp", "ets", "cts") + self.stdlib_path=path.join(self.test_root, "stdlib") + self.compiler_args = f"--stdlib={self.stdlib_path} --gen-stdlib=false --extension=ets --opt_level=0" + + stdlib_file = Path(output_path) + if not stdlib_file.exists(): + self._generate_ets_stdlib(self.stdlib_path, output_path) + self.test_env = TestEnv( + args=args, + cmd_env=self.cmd_env, + compiler=self.compiler, + runtime=self.runtime, + compiler_args = self.compiler_args, + runtime_args=self.runtime_args, + stdlib_path=self.stdlib_path, + bytecode_path=self.bytecode_path + ) + + def _generate_ets_stdlib(self, stdlib_path : str, output_path : str): + dummy_test=path.join(self.test_root, "stdlib/std/core/Boolean.ets") + options=f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" + cmd_string=f"{self.compiler} {options}" + args=shlex.split(cmd_string) + process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) - def _generate_ets_stdlib(): - pass + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Cannot generate ETS stdlib : timeout") - + if process.returncode != 0: + raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - \ No newline at end of file diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index 55b0dafbe..aba767ab7 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -1,3 +1,6 @@ +import os.path +import shlex +import subprocess from runner_base import correct_path, get_test_id from runner_ets import RunnerETS from test_ets_cts import TestETS_CTS @@ -6,9 +9,34 @@ from test_ets_cts import TestETS_CTS class RunnerETS_CTS(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-cts") + self.etc_cts_generated_code="/tmp/cts-generated" + self.code_generator="scripts/formatChecker/generate.py" + self.python_engine="python3" + self._generate_ets_code_from_template() + self.test_root=self.etc_cts_generated_code + self.add_directory(self.etc_cts_generated_code, "ets", []) + + + def _generate_ets_code_from_template(self): + cmd_string = f"{self.python_engine} {self.code_generator} ./CTS {self.etc_cts_generated_code}" + args=shlex.split(cmd_string) + work_dir = os.path.join(self.test_root, "tests") + process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_dir, env=self.cmd_env) + + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Cannot generate CTS code from template : timeout") + + if process.returncode != 0: + raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) + - def _generate_ets_code_from_template(): - pass + def create_test(self, test_file, flags, is_ignored): + test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) + test.ignored = is_ignored + return test diff --git a/test/runner/starter.py b/test/runner/starter.py index 553faa5f3..599ee95fd 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -44,7 +44,7 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - parser.add_argument('--ets-cts', action='store_true', dest='ets_sts', + parser.add_argument('--ets-cts', action='store_true', dest='ets_cts', default=False, help='run test agaisnt ETS specification') parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', default=False, help='run test agaisnt ETS specification') diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 8ec62c3d5..e75bf4680 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -10,8 +10,16 @@ from test_base import Test class TestETS(Test): - def __init__(self, test_env, test_path, flags, test_id, update_expected): + def __init__(self, test_path, test_env, flags, test_id, update_expected): Test.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further - self.fail_kind = None \ No newline at end of file + self.fail_kind = None + + def run_compiler(self): + options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --output={} --opt-level=0 {self.test_path}" + self.test_env.compiler + pass + + def run_runtime(): + pass \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py index 80e421fa8..8933ad052 100644 --- a/test/runner/test_ets_cts.py +++ b/test/runner/test_ets_cts.py @@ -3,9 +3,9 @@ from os import path, makedirs from configuration_kind import ConfigurationKind from test_ets import TestETS from utils import purify - +# test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) class TestETS_CTS(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + def __init__(self, test_path, test_env, flags, test_id=None, update_expected=False): TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) self.tmp_dir = path.join(path.sep, "tmp", "cts") makedirs(self.tmp_dir, exist_ok=True) -- Gitee From 6cfe453826db7cdf53ca3900a691c5827107fdec Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Wed, 19 Oct 2022 16:16:50 +0300 Subject: [PATCH 05/36] execution of CTS was debugged --- test/runner/runner_base.py | 1 + test/runner/runner_ets.py | 55 +++++++++++++++++------------- test/runner/runner_ets_cts.py | 5 +-- test/runner/test_ets.py | 64 +++++++++++++++++++++++++++++++---- test/runner/test_ets_cts.py | 2 +- 5 files changed, 93 insertions(+), 34 deletions(-) diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 506e7c46a..0aadc492a 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -180,6 +180,7 @@ class Runner: ) ))) + def create_test(self, test_file, flags, is_ignored): pass diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 0f028d6e3..11f7ef6e7 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -28,40 +28,47 @@ INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" class RunnerETS(Runner): - def __init__(self, args, name:str): + def __init__(self, args, name: str): Runner.__init__(self, args, name) - output_path = path.join(self.stdlib_path, self.stdlib_name) + self.stdlib_path = path.join(self.test_root, "stdlib") + self.stdlib_name = "etsstdlib.abc" + stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) self.cmd_env = environ.copy() - self.test_root=args.test_root - self.compiler=path.join(self.build_dir, "bin", "es2panda") + self.test_root = args.test_root + self.compiler = path.join(self.build_dir, "bin", "es2panda") self.runtime = path.join(self.build_dir, "bin", "ark") - self.runtime_args = f"--boot-panda={output_path} --load_runtime=ets" self.cmd_env = environ.copy() - self.stdlib_name="etsstdlib.abc" - self.bytecode_path=path.join("/tmp", "ets", "cts") - self.stdlib_path=path.join(self.test_root, "stdlib") - self.compiler_args = f"--stdlib={self.stdlib_path} --gen-stdlib=false --extension=ets --opt_level=0" + self.stdlib_name = "etsstdlib.abc" + self.stdlib_path = path.join(self.test_root, "stdlib") - stdlib_file = Path(output_path) + + stdlib_file = Path(stdlib_output_path) if not stdlib_file.exists(): - self._generate_ets_stdlib(self.stdlib_path, output_path) + self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) self.test_env = TestEnv( args=args, cmd_env=self.cmd_env, - compiler=self.compiler, + es2panda =self.compiler, runtime=self.runtime, - compiler_args = self.compiler_args, - runtime_args=self.runtime_args, - stdlib_path=self.stdlib_path, - bytecode_path=self.bytecode_path + runtime_args="", + arkaout="", + aot_args="", + ark_quick="", + quick_args="", + conf_kind="", + cmd_prefix="" ) - def _generate_ets_stdlib(self, stdlib_path : str, output_path : str): - dummy_test=path.join(self.test_root, "stdlib/std/core/Boolean.ets") - options=f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" - cmd_string=f"{self.compiler} {options}" - args=shlex.split(cmd_string) - process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) + self.test_env.stdlib_path = self.stdlib_path + self.test_env.stdlib_output_path = stdlib_output_path + + def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): + dummy_test = path.join(self.test_root, "stdlib/std/core/Boolean.ets") + options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" + cmd_string = f"{self.compiler} {options}" + args = shlex.split(cmd_string) + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) try: out, err = process.communicate(timeout=600) @@ -70,5 +77,5 @@ class RunnerETS(Runner): raise Exception("Cannot generate ETS stdlib : timeout") if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - + raise Exception("Generating ETS stdlib fail %d" % + (process.returncode)) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index aba767ab7..0b782f0c3 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -3,7 +3,7 @@ import shlex import subprocess from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_cts import TestETS_CTS +from test_ets import TestETS class RunnerETS_CTS(RunnerETS): @@ -13,6 +13,7 @@ class RunnerETS_CTS(RunnerETS): self.code_generator="scripts/formatChecker/generate.py" self.python_engine="python3" self._generate_ets_code_from_template() + self.test_root_backup = self.test_root self.test_root=self.etc_cts_generated_code self.add_directory(self.etc_cts_generated_code, "ets", []) @@ -34,7 +35,7 @@ class RunnerETS_CTS(RunnerETS): def create_test(self, test_file, flags, is_ignored): - test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) + test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) test.ignored = is_ignored return test diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index e75bf4680..b2568ec91 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -2,6 +2,7 @@ import subprocess import sys from os import path, remove from typing import List +import shlex from configuration_kind import ConfigurationKind from fail_kind import FailKind @@ -10,16 +11,65 @@ from test_base import Test class TestETS(Test): - def __init__(self, test_path, test_env, flags, test_id, update_expected): + def __init__(self, test_env, test_path, flags, test_id, update_expected=False): Test.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None + self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" + self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" + self.common_ets_runtime_options = test_env.runtime_args + self.ets_main_entry_point="ETSGLOBAL::main" + self.bytecode_path = path.join("/tmp", "ets", "cts") - def run_compiler(self): - options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --output={} --opt-level=0 {self.test_path}" - self.test_env.compiler - pass + def do_run(self): + test_basename = path.basename(self.path) + is_negative_compilation = False + is_negative_runtime = False + if test_basename.startswith("n."): + is_negative_compilation = True + if test_basename.startswith("n.e."): + is_negative_runtime = True - def run_runtime(): - pass \ No newline at end of file + compiled_file, rc = self._run_compiler(is_negative_compilation) + if rc > 0 : + return ; + + if not is_negative_compilation: + self._run_runtime(compiled_file, is_negative_runtime) + + def _run_compiler(self, is_negative = False) -> str: + test_basename = path.basename(self.path) + test_abc = f"{test_basename}.abc" + output_path = path.join(self.bytecode_path, "abc", test_abc) + ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" + args = shlex.split(compiler_cmd_string) + rc = self._run_process_from_python(args, is_negative) + return (output_path,rc) + + + def _run_runtime(self, abc_file:str, is_negative:bool): + runtime_cmd_string = f"{self.common_ets_runtime_options} {abc_file} {self.ets_main_entry_point}" + args = shlex.split(runtime_cmd_string) + runtime_rc = self._run_process_from_python(args, is_negative) + return runtime_rc + + def _run_process_from_python(self, args:list, is_negative:bool)->None: + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) + + if process.returncode != 0: + if is_negative: + return 0 + else: + return 1 + + + + \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py index 8933ad052..ff1e7f510 100644 --- a/test/runner/test_ets_cts.py +++ b/test/runner/test_ets_cts.py @@ -5,7 +5,7 @@ from test_ets import TestETS from utils import purify # test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) class TestETS_CTS(TestETS): - def __init__(self, test_path, test_env, flags, test_id=None, update_expected=False): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) self.tmp_dir = path.join(path.sep, "tmp", "cts") makedirs(self.tmp_dir, exist_ok=True) -- Gitee From 64ba088006fc1cca55980a950a8871b0d8c3bb44 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Wed, 19 Oct 2022 19:30:16 +0300 Subject: [PATCH 06/36] hot fixes for execution of CTS was debugged --- test/runner/test_ets.py | 52 +++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index b2568ec91..4c6353437 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -1,6 +1,6 @@ import subprocess import sys -from os import path, remove +from os import path, makedirs from typing import List import shlex @@ -18,9 +18,9 @@ class TestETS(Test): self.fail_kind = None self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" - self.common_ets_runtime_options = test_env.runtime_args self.ets_main_entry_point="ETSGLOBAL::main" - self.bytecode_path = path.join("/tmp", "ets", "cts") + self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") + makedirs(self.bytecode_path, exist_ok=True) def do_run(self): test_basename = path.basename(self.path) @@ -33,42 +33,54 @@ class TestETS(Test): compiled_file, rc = self._run_compiler(is_negative_compilation) if rc > 0 : - return ; + return self; + else: + print("Compilation Passed") if not is_negative_compilation: - self._run_runtime(compiled_file, is_negative_runtime) + runtime_rc = self._run_runtime(compiled_file, is_negative_runtime) + if runtime_rc == 0: + print ("Runtime Passed") + else: + print ("Runtime Failed") + return self - def _run_compiler(self, is_negative = False) -> str: + def _run_compiler(self, is_negative:bool) -> str: test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" - output_path = path.join(self.bytecode_path, "abc", test_abc) + output_path = path.join(self.bytecode_path, test_abc) ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - rc = self._run_process_from_python(args, is_negative) - return (output_path,rc) + compile_rc = self._run_process_from_python(args) + if is_negative and compile_rc != 0: + return (output_path, 0) + elif is_negative and compile_rc == 0: + return (output_path, 1) + else: + return(output_path, 0) def _run_runtime(self, abc_file:str, is_negative:bool): - runtime_cmd_string = f"{self.common_ets_runtime_options} {abc_file} {self.ets_main_entry_point}" + runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - runtime_rc = self._run_process_from_python(args, is_negative) - return runtime_rc + runtime_rc = self._run_process_from_python(args) + if is_negative and runtime_rc != 0: + return 0 + elif is_negative and runtime_rc == 0: + return 1 + else: + return 0 - def _run_process_from_python(self, args:list, is_negative:bool)->None: + def _run_process_from_python(self, args:list): process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - out, err = process.communicate(timeout=600) + out, err = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - if process.returncode != 0: - if is_negative: - return 0 - else: - return 1 + return process.returncode != 0 -- Gitee From c2078c10705b89ddf1dd86dfa0b12921c6453cdc Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 11:27:12 +0300 Subject: [PATCH 07/36] update for report generating --- test/runner/test_ets.py | 45 ++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 4c6353437..2a3a52bdf 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -31,18 +31,13 @@ class TestETS(Test): if test_basename.startswith("n.e."): is_negative_runtime = True - compiled_file, rc = self._run_compiler(is_negative_compilation) - if rc > 0 : + self.compile_report, compiled_file = self._run_compiler(is_negative_compilation) + if self.compile_report.return_code > 0 : return self; - else: - print("Compilation Passed") if not is_negative_compilation: - runtime_rc = self._run_runtime(compiled_file, is_negative_runtime) - if runtime_rc == 0: - print ("Runtime Passed") - else: - print ("Runtime Failed") + self.runtime_report = self._run_runtime(compiled_file, is_negative_runtime) + return self def _run_compiler(self, is_negative:bool) -> str: @@ -52,35 +47,47 @@ class TestETS(Test): ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - compile_rc = self._run_process_from_python(args) + output_content, error_content, compile_rc = self._run_process_from_python(args) + compile_report = TestReport( + output=output_content, + error = error_content, + return_code = 0 + ) if is_negative and compile_rc != 0: - return (output_path, 0) + return (compile_report, output_path) elif is_negative and compile_rc == 0: - return (output_path, 1) + compile_report.return_code = 1 + return (compile_report, output_path) else: - return(output_path, 0) + return(compile_report, output_path) def _run_runtime(self, abc_file:str, is_negative:bool): runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - runtime_rc = self._run_process_from_python(args) + output_content, error_content, runtime_rc = self._run_process_from_python(args) + runtime_report = TestReport( + output=output_content, + error = error_content, + return_code = 0 + ) if is_negative and runtime_rc != 0: - return 0 + return runtime_report elif is_negative and runtime_rc == 0: - return 1 + runtime_report.return_code = 1 + return runtime_report else: - return 0 + return runtime_report def _run_process_from_python(self, args:list): process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: - out, err = process.communicate(timeout=60) + output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - return process.returncode != 0 + return (output_content, err_content,process.returncode) -- Gitee From b3c23c9f68e1426514a9d55e9139ae6d50311957 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 12:44:48 +0300 Subject: [PATCH 08/36] report generating was debugged --- test/runner/runner_ets.py | 5 +- test/runner/runner_ets_stdlib.py | 5 +- .../{runner_js.py => runner_file_based.py} | 2 +- test/runner/runner_js_hermes.py | 6 +- test/runner/runner_js_parser.py | 4 +- test/runner/runner_js_test262.py | 6 +- test/runner/test_ets.py | 80 +++++++++++-------- test/runner/test_ets_cts.py | 13 --- test/runner/test_ets_stdlib.py | 13 --- 9 files changed, 61 insertions(+), 73 deletions(-) rename test/runner/{runner_js.py => runner_file_based.py} (99%) delete mode 100644 test/runner/test_ets_cts.py delete mode 100644 test/runner/test_ets_stdlib.py diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 11f7ef6e7..f0720a5f9 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -13,6 +13,7 @@ from fail_kind import FailKind from params import TestEnv from report_format import ReportFormat from runner_base import Runner +from runner_file_based import RunnerFileBased from utils import write_2_file INDEX_TITLE = "${Title}" @@ -27,9 +28,9 @@ INDEX_TEST_NAME = "${TestName}" INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" -class RunnerETS(Runner): +class RunnerETS(RunnerFileBased): def __init__(self, args, name: str): - Runner.__init__(self, args, name) + RunnerFileBased.__init__(self, args, name) self.stdlib_path = path.join(self.test_root, "stdlib") self.stdlib_name = "etsstdlib.abc" stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index cf801d317..c8ebbd373 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,15 +1,12 @@ from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_stdlib import TestETS_STDLIB class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): - RunnerETS.__init__(self, args, "ets-cts") - - def _generate_ets_code_from_template(): pass + \ No newline at end of file diff --git a/test/runner/runner_js.py b/test/runner/runner_file_based.py similarity index 99% rename from test/runner/runner_js.py rename to test/runner/runner_file_based.py index 8cefcaef6..f91eece86 100644 --- a/test/runner/runner_js.py +++ b/test/runner/runner_file_based.py @@ -24,7 +24,7 @@ INDEX_TEST_NAME = "${TestName}" INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" -class RunnerJS(Runner): +class RunnerFileBased(Runner): def __init__(self, args, name): Runner.__init__(self, args, name) self.cmd_env = environ.copy() diff --git a/test/runner/runner_js_hermes.py b/test/runner/runner_js_hermes.py index 79cf36021..9c20648b9 100644 --- a/test/runner/runner_js_hermes.py +++ b/test/runner/runner_js_hermes.py @@ -1,12 +1,12 @@ from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_hermes import TestJSHermes from util_hermes import UtilHermes -class RunnerJSHermes(RunnerJS): +class RunnerJSHermes(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "hermes") + RunnerFileBased.__init__(self, args, "hermes") self.collect_excluded_test_lists() self.collect_ignored_test_lists() diff --git a/test/runner/runner_js_parser.py b/test/runner/runner_js_parser.py index 2a134867c..d10fe8bef 100644 --- a/test/runner/runner_js_parser.py +++ b/test/runner/runner_js_parser.py @@ -1,11 +1,11 @@ from os import path from runner_base import get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_parser import TestJSParser -class RunnerJSParser(RunnerJS): +class RunnerJSParser(RunnerFileBased): def __init__(self, args): super(RunnerJSParser, self).__init__(args, "parser-js") diff --git a/test/runner/runner_js_test262.py b/test/runner/runner_js_test262.py index 21a880723..d16af1f07 100644 --- a/test/runner/runner_js_test262.py +++ b/test/runner/runner_js_test262.py @@ -1,14 +1,14 @@ from os import path from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_test262 import TestJSTest262 from util_test262 import UtilTest262 -class RunnerJSTest262(RunnerJS): +class RunnerJSTest262(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "test262-ark") + RunnerFileBased.__init__(self, args, "test262-ark") self.ignored_name_prefix = "test262" self.collect_excluded_test_lists(test_name=self.ignored_name_prefix) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 2a3a52bdf..22aee1fa9 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -12,13 +12,14 @@ from test_base import Test class TestETS(Test): def __init__(self, test_env, test_path, flags, test_id, update_expected=False): - Test.__init__(self, test_env, test_path, flags, test_id, update_expected) + Test.__init__(self, test_env, test_path, + flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" - self.ets_main_entry_point="ETSGLOBAL::main" + self.ets_main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") makedirs(self.bytecode_path, exist_ok=True) @@ -31,45 +32,62 @@ class TestETS(Test): if test_basename.startswith("n.e."): is_negative_runtime = True - self.compile_report, compiled_file = self._run_compiler(is_negative_compilation) - if self.compile_report.return_code > 0 : - return self; + self.report, compiled_file = self._run_compiler(is_negative_compilation) + if self.report.return_code > 0: + self.passed=False + self.fail_kind= FailKind.ES2PANDA_FAIL + return self + elif self.report.return_code == 0 and is_negative_compilation: + self.passed = True + return self + else: + pass - if not is_negative_compilation: - self.runtime_report = self._run_runtime(compiled_file, is_negative_runtime) + if not is_negative_compilation: + self.report = self._run_runtime(compiled_file, is_negative_runtime) + if self.report.return_code > 0: + self.passed = False + self.fail_kind = FailKind.RUNTIME_FAIL + else: + self.passed = True return self - def _run_compiler(self, is_negative:bool) -> str: + def _run_compiler(self, is_negative: bool) -> str: test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - output_content, error_content, compile_rc = self._run_process_from_python(args) - compile_report = TestReport( - output=output_content, - error = error_content, - return_code = 0 + output_content, error_content, compile_rc = self._run_process_from_python( + args) + compile_report_succes = TestReport( + output=output_content.decode('UTF-8'), + error=error_content.decode('UTF-8'), + return_code=0 + ) + compile_report_fail = TestReport( + output=output_content.decode('UTF-8'), + error=error_content.decode('UTF-8'), + return_code=1 ) if is_negative and compile_rc != 0: - return (compile_report, output_path) + return (compile_report_succes, output_path) elif is_negative and compile_rc == 0: - compile_report.return_code = 1 - return (compile_report, output_path) + return (compile_report_fail, output_path) else: - return(compile_report, output_path) - + return (compile_report_succes, output_path) - def _run_runtime(self, abc_file:str, is_negative:bool): - runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" + def _run_runtime(self, abc_file: str, is_negative: bool): + runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - output_content, error_content, runtime_rc = self._run_process_from_python(args) + output_content, error_content, runtime_rc = self._run_process_from_python( + args) runtime_report = TestReport( output=output_content, - error = error_content, - return_code = 0 + error=error_content, + return_code=0 ) if is_negative and runtime_rc != 0: return runtime_report @@ -79,16 +97,14 @@ class TestETS(Test): else: return runtime_report - def _run_process_from_python(self, args:list): - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + def _run_process_from_python(self, args: list): + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() - raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - - return (output_content, err_content,process.returncode) - - + raise Exception( + "Hmm .... Task completed %s : timeout" % (self.test_id)) - \ No newline at end of file + return (output_content, err_content, process.returncode) diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py deleted file mode 100644 index ff1e7f510..000000000 --- a/test/runner/test_ets_cts.py +++ /dev/null @@ -1,13 +0,0 @@ -from os import path, makedirs - -from configuration_kind import ConfigurationKind -from test_ets import TestETS -from utils import purify -# test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) -class TestETS_CTS(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): - TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) - self.tmp_dir = path.join(path.sep, "tmp", "cts") - makedirs(self.tmp_dir, exist_ok=True) - self.util = self.test_env.util - diff --git a/test/runner/test_ets_stdlib.py b/test/runner/test_ets_stdlib.py deleted file mode 100644 index dd1b3fb07..000000000 --- a/test/runner/test_ets_stdlib.py +++ /dev/null @@ -1,13 +0,0 @@ -from os import path, makedirs - -from configuration_kind import ConfigurationKind -from test_ets import TestETS -from utils import purify - -class TestETS_STDLIB(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): - TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) - self.tmp_dir = path.join(path.sep, "tmp", "cts") - makedirs(self.tmp_dir, exist_ok=True) - self.util = self.test_env.util - -- Gitee From 0c113fbe615c2e6f2092276eed553d172751af86 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 18:09:18 +0300 Subject: [PATCH 09/36] update for stdlib --- test/runner/runner.py | 4 -- test/runner/runner_ets_cts.py | 44 -------------- test/runner/runner_ets_stdlib.py | 15 ++++- test/runner/test_ets.py | 98 +++++++++++++++----------------- 4 files changed, 58 insertions(+), 103 deletions(-) delete mode 100644 test/runner/runner_ets_cts.py diff --git a/test/runner/runner.py b/test/runner/runner.py index 74ca8960a..995ced502 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -4,7 +4,6 @@ from dotenv import load_dotenv from runner_js_hermes import RunnerJSHermes from runner_js_parser import RunnerJSParser from runner_js_test262 import RunnerJSTest262 -from runner_ets_cts import RunnerETS_CTS from runner_ets_stdlib import RunnerETS_STDLIB from starter import get_args @@ -29,9 +28,6 @@ def main(): if args.hermes: runners.append(RunnerJSHermes(args)) - if args.ets_cts: - runners.append(RunnerETS_CTS(args)) - if args.ets_stdlib: runners.append(RunnerETS_STDLIB(args)) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py deleted file mode 100644 index 0b782f0c3..000000000 --- a/test/runner/runner_ets_cts.py +++ /dev/null @@ -1,44 +0,0 @@ -import os.path -import shlex -import subprocess -from runner_base import correct_path, get_test_id -from runner_ets import RunnerETS -from test_ets import TestETS - - -class RunnerETS_CTS(RunnerETS): - def __init__(self, args): - RunnerETS.__init__(self, args, "ets-cts") - self.etc_cts_generated_code="/tmp/cts-generated" - self.code_generator="scripts/formatChecker/generate.py" - self.python_engine="python3" - self._generate_ets_code_from_template() - self.test_root_backup = self.test_root - self.test_root=self.etc_cts_generated_code - self.add_directory(self.etc_cts_generated_code, "ets", []) - - - def _generate_ets_code_from_template(self): - cmd_string = f"{self.python_engine} {self.code_generator} ./CTS {self.etc_cts_generated_code}" - args=shlex.split(cmd_string) - work_dir = os.path.join(self.test_root, "tests") - process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_dir, env=self.cmd_env) - - try: - out, err = process.communicate(timeout=600) - except subprocess.TimeoutExpired: - process.kill() - raise Exception("Cannot generate CTS code from template : timeout") - - if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - - - def create_test(self, test_file, flags, is_ignored): - test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) - test.ignored = is_ignored - return test - - - - \ No newline at end of file diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index c8ebbd373..f2058b6ed 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,10 +1,21 @@ -from runner_base import correct_path, get_test_id +import shlex +import os +import subprocess from runner_ets import RunnerETS +from test_ets import TestETS +from runner_base import get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): - pass + RunnerETS.__init__(self, args, "ets-stdlib") + self.add_directory(self.root_test, "ets", []) + + + def create_test(self, test_file, flags, is_ignored): + test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) + test.ignored = is_ignored + return test diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 22aee1fa9..3362b6d8e 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -2,6 +2,7 @@ import subprocess import sys from os import path, makedirs from typing import List +from typing import Tuple import shlex from configuration_kind import ConfigurationKind @@ -20,82 +21,74 @@ class TestETS(Test): self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" self.ets_main_entry_point = "ETSGLOBAL::main" - self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") + self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) def do_run(self): test_basename = path.basename(self.path) - is_negative_compilation = False - is_negative_runtime = False + is_negative_test = False if test_basename.startswith("n."): - is_negative_compilation = True - if test_basename.startswith("n.e."): - is_negative_runtime = True + is_negative_test = True - self.report, compiled_file = self._run_compiler(is_negative_compilation) - if self.report.return_code > 0: + report, compiled_file, fail_kind = self._run_compiler() + if report.return_code > 0: self.passed=False - self.fail_kind= FailKind.ES2PANDA_FAIL + self.fail_kind= fail_kind + self.report = report return self - elif self.report.return_code == 0 and is_negative_compilation: + + self.report = self._run_runtime(compiled_file, is_negative_test) + if self.report.return_code > 0 and is_negative_test: self.passed = True return self + elif self.report.return_code > 0: + self.passed = False + self.fail_kind = FailKind.RUNTIME_FAIL + return self else: - pass - - if not is_negative_compilation: - self.report = self._run_runtime(compiled_file, is_negative_runtime) - if self.report.return_code > 0: - self.passed = False - self.fail_kind = FailKind.RUNTIME_FAIL - else: - self.passed = True + self.passed = True + return self - return self - def _run_compiler(self, is_negative: bool) -> str: + def _run_compiler(self) -> Tuple[TestReport,str,FailKind]: + fail_kind = None test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - output_content, error_content, compile_rc = self._run_process_from_python( - args) - compile_report_succes = TestReport( - output=output_content.decode('UTF-8'), - error=error_content.decode('UTF-8'), - return_code=0 - ) - compile_report_fail = TestReport( - output=output_content.decode('UTF-8'), - error=error_content.decode('UTF-8'), - return_code=1 + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + try: + stdout_content, stderr_content = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + fail_kind = FailKind.ES2PANDA_TIMEOUT + + report = TestReport( + output=stdout_content, + error=stderr_content, + return_code=process.returncode ) - if is_negative and compile_rc != 0: - return (compile_report_succes, output_path) - elif is_negative and compile_rc == 0: - return (compile_report_fail, output_path) - else: - return (compile_report_succes, output_path) + return (report, output_path, fail_kind) def _run_runtime(self, abc_file: str, is_negative: bool): + fail_kind = None runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - output_content, error_content, runtime_rc = self._run_process_from_python( - args) + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + try: + stdout_content, stderr_content = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + fail_kind = FailKind.RUNTIME_TIMEOUT + runtime_report = TestReport( - output=output_content, - error=error_content, - return_code=0 + output=stdout_content, + error=stderr_content, + return_code=process.returncode ) - if is_negative and runtime_rc != 0: - return runtime_report - elif is_negative and runtime_rc == 0: - runtime_report.return_code = 1 - return runtime_report - else: - return runtime_report + return runtime_report, fail_kind def _run_process_from_python(self, args: list): process = subprocess.Popen( @@ -104,7 +97,6 @@ class TestETS(Test): output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() - raise Exception( - "Hmm .... Task completed %s : timeout" % (self.test_id)) + fail_kind = FailKind.PROCESS_TIMEOUT - return (output_content, err_content, process.returncode) + return (output_content, err_content, process.returncode, fail_kind) -- Gitee From 4540301fa7425cf6d68a7cd43bf9553f6bdce42d Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 18:24:27 +0300 Subject: [PATCH 10/36] update for ets-stdlib.not debugged --- test/runner/runner_ets.py | 4 +++- test/runner/starter.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index f0720a5f9..68c37d7b9 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -64,7 +64,7 @@ class RunnerETS(RunnerFileBased): self.test_env.stdlib_output_path = stdlib_output_path def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): - dummy_test = path.join(self.test_root, "stdlib/std/core/Boolean.ets") + dummy_test = path.join(self.test_root, "std/dummy.ets") options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" cmd_string = f"{self.compiler} {options}" args = shlex.split(cmd_string) @@ -77,6 +77,8 @@ class RunnerETS(RunnerFileBased): process.kill() raise Exception("Cannot generate ETS stdlib : timeout") + print("StdOut : " + out.decode('UTF-8')) + print("StdErr : " + err.decode('UTF-8')) if process.returncode != 0: raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) diff --git a/test/runner/starter.py b/test/runner/starter.py index 599ee95fd..5853fc662 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -44,8 +44,6 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - parser.add_argument('--ets-cts', action='store_true', dest='ets_cts', - default=False, help='run test agaisnt ETS specification') parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', default=False, help='run test agaisnt ETS specification') parser.add_argument( -- Gitee From 792a7a3225ffae03373503a6d006b8a66b515cb0 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Fri, 21 Oct 2022 16:05:59 +0300 Subject: [PATCH 11/36] support for stdlib test is ready for review 1 --- test/runner/runner_ets.py | 57 +++++++++++++++----------------- test/runner/runner_ets_stdlib.py | 3 +- test/runner/test_ets.py | 52 +++++++++++++---------------- 3 files changed, 51 insertions(+), 61 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 68c37d7b9..68afaa391 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -1,11 +1,6 @@ -import subprocess -import sys -from datetime import datetime -from glob import glob -from os import path, environ, makedirs -from typing import List +from cProfile import run +from os import path, environ from pathlib import Path -import shlex from configuration_kind import ConfigurationKind @@ -39,19 +34,21 @@ class RunnerETS(RunnerFileBased): self.compiler = path.join(self.build_dir, "bin", "es2panda") self.runtime = path.join(self.build_dir, "bin", "ark") self.cmd_env = environ.copy() - self.stdlib_name = "etsstdlib.abc" - self.stdlib_path = path.join(self.test_root, "stdlib") + self.stdlib_path = path.join(self.build_dir, "etsstdlib.abc") + self.stdlib_src_path = path.join(self.test_root, "stdlib") + self._check_binary_artefacts() + stdlib_file = Path(stdlib_output_path) - if not stdlib_file.exists(): - self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) + # if not stdlib_file.exists(): + # self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) self.test_env = TestEnv( args=args, cmd_env=self.cmd_env, es2panda =self.compiler, runtime=self.runtime, - runtime_args="", + runtime_args=f"--boot-panda-files={self.stdlib_path} --load-runtimes=ets", arkaout="", aot_args="", ark_quick="", @@ -60,25 +57,23 @@ class RunnerETS(RunnerFileBased): cmd_prefix="" ) - self.test_env.stdlib_path = self.stdlib_path - self.test_env.stdlib_output_path = stdlib_output_path + self.test_env.es2panda_args= f"--stdlib={self.stdlib_src_path} --gen-stdlib=false --extension=ets --opt-level=0" + + + def _check_binary_artefacts(self) : + compiler_path_obj = Path(self.compiler) + runtime_path_obj = Path(self.runtime) + stdlib_path_obj = Path(self.stdlib_path) + stdlib_src_path_obj = Path(self.stdlib_src_path) + + if not compiler_path_obj.is_file(): + raise Exception("Hmm ... ETS Compiler was not found") - def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): - dummy_test = path.join(self.test_root, "std/dummy.ets") - options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" - cmd_string = f"{self.compiler} {options}" - args = shlex.split(cmd_string) - process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) + if not runtime_path_obj.is_file(): + raise Exception("Hmm ... ETS Runtime was not found") - try: - out, err = process.communicate(timeout=600) - except subprocess.TimeoutExpired: - process.kill() - raise Exception("Cannot generate ETS stdlib : timeout") + if not stdlib_path_obj.is_file(): + raise Exception("Hmm ... standart library abc file not found") - print("StdOut : " + out.decode('UTF-8')) - print("StdErr : " + err.decode('UTF-8')) - if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % - (process.returncode)) + if not stdlib_src_path_obj.is_dir(): + raise Exception("Hmm ... Source code of standart library was not found") \ No newline at end of file diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index f2058b6ed..6187e4782 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -9,7 +9,8 @@ from runner_base import get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-stdlib") - self.add_directory(self.root_test, "ets", []) + self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") + self.add_directory(self.stdlib_test_path, "ets", []) def create_test(self, test_file, flags, is_ignored): diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 3362b6d8e..172d4f1d4 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -18,8 +18,6 @@ class TestETS(Test): # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None - self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" - self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" self.ets_main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) @@ -31,19 +29,18 @@ class TestETS(Test): is_negative_test = True report, compiled_file, fail_kind = self._run_compiler() - if report.return_code > 0: - self.passed=False - self.fail_kind= fail_kind - self.report = report + self.fail_kind= fail_kind + self.report = report + if report.return_code != 0: + self.passed = False return self - self.report = self._run_runtime(compiled_file, is_negative_test) - if self.report.return_code > 0 and is_negative_test: + self.report, self.fail_kind = self._run_runtime(compiled_file) + if self.report.return_code != 0 and is_negative_test: self.passed = True return self - elif self.report.return_code > 0: + elif self.report.return_code != 0: self.passed = False - self.fail_kind = FailKind.RUNTIME_FAIL return self else: self.passed = True @@ -55,9 +52,10 @@ class TestETS(Test): test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + ets_compiler_options = f"{self.test_env.es2panda_args} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) + self.log_cmd(args) process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: stdout_content, stderr_content = process.communicate(timeout=60) @@ -65,17 +63,21 @@ class TestETS(Test): process.kill() fail_kind = FailKind.ES2PANDA_TIMEOUT + if process.returncode != 0: + fail_kind = FailKind.ES2PANDA_FAIL + report = TestReport( - output=stdout_content, - error=stderr_content, + output=stdout_content.decode('UTF-8'), + error=stderr_content.decode('UTF-8'), return_code=process.returncode ) return (report, output_path, fail_kind) - def _run_runtime(self, abc_file: str, is_negative: bool): + def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: fail_kind = None - runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" + runtime_cmd_string = f"{self.test_env.runtime} {self.test_env.runtime_args} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) + self.log_cmd(args) process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: stdout_content, stderr_content = process.communicate(timeout=60) @@ -83,20 +85,12 @@ class TestETS(Test): process.kill() fail_kind = FailKind.RUNTIME_TIMEOUT + if process.returncode != 0: + fail_kind = FailKind.RUNTIME_FAIL + runtime_report = TestReport( - output=stdout_content, - error=stderr_content, + output=stdout_content.decode('UTF-8'), + error=stderr_content.decode('UTF-8'), return_code=process.returncode ) - return runtime_report, fail_kind - - def _run_process_from_python(self, args: list): - process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - output_content, err_content = process.communicate(timeout=60) - except subprocess.TimeoutExpired: - process.kill() - fail_kind = FailKind.PROCESS_TIMEOUT - - return (output_content, err_content, process.returncode, fail_kind) + return runtime_report, fail_kind \ No newline at end of file -- Gitee From 92e0c76d776cd9dcbaac4727a3d287b97d96efff Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 14:53:09 +0300 Subject: [PATCH 12/36] add excluded list processing --- test/ets-stdlib-excluded.txt | 3 +++ test/runner/runner_base.py | 1 + test/runner/runner_ets_stdlib.py | 5 ++++- test/runner/test_ets.py | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 test/ets-stdlib-excluded.txt diff --git a/test/ets-stdlib-excluded.txt b/test/ets-stdlib-excluded.txt new file mode 100644 index 000000000..477b26172 --- /dev/null +++ b/test/ets-stdlib-excluded.txt @@ -0,0 +1,3 @@ +tests/stdlib/std/math/sqrt-negative-01.ets +tests/stdlib/std/math/sqrt-positive-03.ets +tests/stdlib/std/math/sqrt-positive-04.ets \ No newline at end of file diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 0aadc492a..a1c6fbbd2 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -133,6 +133,7 @@ class Runner: # Read excluded_lists and load list of excluded tests def load_excluded_tests(self): + print(self.excluded_lists) self.excluded_tests.update(self.load_tests_from_lists(self.excluded_lists)) self.excluded = len(self.excluded_tests) diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index 6187e4782..97dd11067 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -3,14 +3,17 @@ import os import subprocess from runner_ets import RunnerETS from test_ets import TestETS -from runner_base import get_test_id +from runner_base import correct_path, get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-stdlib") self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") + self.ignored_name_prefix = self.name + self.excluded_lists = [correct_path(self.list_root, f"{self.ignored_name_prefix}-excluded.txt")] self.add_directory(self.stdlib_test_path, "ets", []) + print (self.excluded_lists) def create_test(self, test_file, flags, is_ignored): diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 172d4f1d4..e7d666207 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -71,6 +71,7 @@ class TestETS(Test): error=stderr_content.decode('UTF-8'), return_code=process.returncode ) + return (report, output_path, fail_kind) def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: -- Gitee From 3ea1213209df9ae2c5fc5a0ddfbb52b4a56e5e7f Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 14:29:45 +0300 Subject: [PATCH 13/36] The first ETS test running structure --- test/runner/runner_ets.py | 37 +++++++++++++++++++++++++++++++++++ test/runner/runner_ets_cts.py | 12 ++++++++++++ test/runner/test_ets.py | 17 ++++++++++++++++ test/runner/test_ets_cts.py | 13 ++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 test/runner/runner_ets.py create mode 100644 test/runner/runner_ets_cts.py create mode 100644 test/runner/test_ets.py create mode 100644 test/runner/test_ets_cts.py diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py new file mode 100644 index 000000000..de3b5bc1c --- /dev/null +++ b/test/runner/runner_ets.py @@ -0,0 +1,37 @@ +import subprocess +import sys +from datetime import datetime +from glob import glob +from os import path, environ, makedirs +from typing import List + +from configuration_kind import ConfigurationKind +from fail_kind import FailKind +from params import TestEnv +from report_format import ReportFormat +from runner_base import Runner +from utils import write_2_file + +INDEX_TITLE = "${Title}" +INDEX_OPTIONS = "${Options}" +INDEX_TOTAL = "${Total}" +INDEX_PASSED = "${Passed}" +INDEX_FAILED = "${Failed}" +INDEX_IGNORED = "${Ignored}" +INDEX_EXCLUDED_LISTS = "${ExcludedThroughLists}" +INDEX_EXCLUDED_OTHER = "${ExcludedByOtherReasons}" +INDEX_TEST_NAME = "${TestName}" +INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" + + +class RunnerETS(Runner): + def __init__(self, args, name): + Runner.__init__(self, args, name) + self.cmd_env = environ.copy() + + def _generate_ets_stdlib(): + pass + + + + \ No newline at end of file diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py new file mode 100644 index 000000000..daa27ad29 --- /dev/null +++ b/test/runner/runner_ets_cts.py @@ -0,0 +1,12 @@ +from runner_base import correct_path, get_test_id +from runner_ets import RunnerETS +from test_ets_cts import TestJSHermes +from util_hermes import UtilHermes + + +class RunnerETSCTS(RunnerJS): + def __init__(self, args): + RunnerETS.__init__(self, args, "hermes") + + + \ No newline at end of file diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py new file mode 100644 index 000000000..8ec62c3d5 --- /dev/null +++ b/test/runner/test_ets.py @@ -0,0 +1,17 @@ +import subprocess +import sys +from os import path, remove +from typing import List + +from configuration_kind import ConfigurationKind +from fail_kind import FailKind +from params import Params, TestReport +from test_base import Test + + +class TestETS(Test): + def __init__(self, test_env, test_path, flags, test_id, update_expected): + Test.__init__(self, test_env, test_path, flags, test_id, update_expected) + # If test fails it contains reason (of FailKind enum) of first failed step + # It's supposed if the first step is failed then no step is executed further + self.fail_kind = None \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py new file mode 100644 index 000000000..80e421fa8 --- /dev/null +++ b/test/runner/test_ets_cts.py @@ -0,0 +1,13 @@ +from os import path, makedirs + +from configuration_kind import ConfigurationKind +from test_ets import TestETS +from utils import purify + +class TestETS_CTS(TestETS): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) + self.tmp_dir = path.join(path.sep, "tmp", "cts") + makedirs(self.tmp_dir, exist_ok=True) + self.util = self.test_env.util + -- Gitee From 6aa3be5bceb5b2b8aea7f92fc6eb74753b47266f Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 14:29:51 +0300 Subject: [PATCH 14/36] add etc cts specific runner --- test/runner/runner_ets_cts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index daa27ad29..db57261d1 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -4,9 +4,13 @@ from test_ets_cts import TestJSHermes from util_hermes import UtilHermes -class RunnerETSCTS(RunnerJS): +class RunnerETSCTS(RunnerETS): def __init__(self, args): - RunnerETS.__init__(self, args, "hermes") + RunnerETS.__init__(self, args, "ets-cts") + + def _generate_ets_code_from_template(): + pass + \ No newline at end of file -- Gitee From 594c788a6e44272d5a8e62d3aec84542f9704102 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 18:32:28 +0300 Subject: [PATCH 15/36] add stdlib structure for runner --- test/runner/runner.py | 8 ++++++++ test/runner/runner_ets_cts.py | 5 ++--- test/runner/runner_ets_stdlib.py | 15 +++++++++++++++ test/runner/starter.py | 8 ++++++-- test/runner/test_ets_stdlib.py | 13 +++++++++++++ 5 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 test/runner/runner_ets_stdlib.py create mode 100644 test/runner/test_ets_stdlib.py diff --git a/test/runner/runner.py b/test/runner/runner.py index 607e8cb25..74ca8960a 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -4,6 +4,8 @@ from dotenv import load_dotenv from runner_js_hermes import RunnerJSHermes from runner_js_parser import RunnerJSParser from runner_js_test262 import RunnerJSTest262 +from runner_ets_cts import RunnerETS_CTS +from runner_ets_stdlib import RunnerETS_STDLIB from starter import get_args @@ -27,6 +29,12 @@ def main(): if args.hermes: runners.append(RunnerJSHermes(args)) + if args.ets_cts: + runners.append(RunnerETS_CTS(args)) + + if args.ets_stdlib: + runners.append(RunnerETS_STDLIB(args)) + failed_tests = 0 for runner in runners: diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index db57261d1..55b0dafbe 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -1,10 +1,9 @@ from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_cts import TestJSHermes -from util_hermes import UtilHermes +from test_ets_cts import TestETS_CTS -class RunnerETSCTS(RunnerETS): +class RunnerETS_CTS(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-cts") diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py new file mode 100644 index 000000000..cf801d317 --- /dev/null +++ b/test/runner/runner_ets_stdlib.py @@ -0,0 +1,15 @@ +from runner_base import correct_path, get_test_id +from runner_ets import RunnerETS +from test_ets_stdlib import TestETS_STDLIB + + +class RunnerETS_STDLIB(RunnerETS): + def __init__(self, args): + RunnerETS.__init__(self, args, "ets-cts") + + def _generate_ets_code_from_template(): + pass + + + + \ No newline at end of file diff --git a/test/runner/starter.py b/test/runner/starter.py index a1f1792f1..553faa5f3 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -21,7 +21,8 @@ def is_file(parser, arg): def check_timeout(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError(f"{value} is an invalid timeout value") + raise argparse.ArgumentTypeError( + f"{value} is an invalid timeout value") return ivalue @@ -43,7 +44,10 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - + parser.add_argument('--ets-cts', action='store_true', dest='ets_sts', + default=False, help='run test agaisnt ETS specification') + parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', + default=False, help='run test agaisnt ETS specification') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), help='directory with test file. If not set the module directory is used') diff --git a/test/runner/test_ets_stdlib.py b/test/runner/test_ets_stdlib.py new file mode 100644 index 000000000..dd1b3fb07 --- /dev/null +++ b/test/runner/test_ets_stdlib.py @@ -0,0 +1,13 @@ +from os import path, makedirs + +from configuration_kind import ConfigurationKind +from test_ets import TestETS +from utils import purify + +class TestETS_STDLIB(TestETS): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) + self.tmp_dir = path.join(path.sep, "tmp", "cts") + makedirs(self.tmp_dir, exist_ok=True) + self.util = self.test_env.util + -- Gitee From 1fbcdef26407345cf7a4994862616497b6a74739 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 18 Oct 2022 18:16:59 +0300 Subject: [PATCH 16/36] add stdlib generating, cts code generating --- test/runner/runner_ets.py | 47 +++++++++++++++++++++++++++++++---- test/runner/runner_ets_cts.py | 32 ++++++++++++++++++++++-- test/runner/starter.py | 2 +- test/runner/test_ets.py | 12 +++++++-- test/runner/test_ets_cts.py | 4 +-- 5 files changed, 85 insertions(+), 12 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index de3b5bc1c..0f028d6e3 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -4,6 +4,9 @@ from datetime import datetime from glob import glob from os import path, environ, makedirs from typing import List +from pathlib import Path +import shlex + from configuration_kind import ConfigurationKind from fail_kind import FailKind @@ -25,13 +28,47 @@ INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" class RunnerETS(Runner): - def __init__(self, args, name): + def __init__(self, args, name:str): Runner.__init__(self, args, name) + output_path = path.join(self.stdlib_path, self.stdlib_name) + self.cmd_env = environ.copy() + self.test_root=args.test_root + self.compiler=path.join(self.build_dir, "bin", "es2panda") + self.runtime = path.join(self.build_dir, "bin", "ark") + self.runtime_args = f"--boot-panda={output_path} --load_runtime=ets" self.cmd_env = environ.copy() + self.stdlib_name="etsstdlib.abc" + self.bytecode_path=path.join("/tmp", "ets", "cts") + self.stdlib_path=path.join(self.test_root, "stdlib") + self.compiler_args = f"--stdlib={self.stdlib_path} --gen-stdlib=false --extension=ets --opt_level=0" + + stdlib_file = Path(output_path) + if not stdlib_file.exists(): + self._generate_ets_stdlib(self.stdlib_path, output_path) + self.test_env = TestEnv( + args=args, + cmd_env=self.cmd_env, + compiler=self.compiler, + runtime=self.runtime, + compiler_args = self.compiler_args, + runtime_args=self.runtime_args, + stdlib_path=self.stdlib_path, + bytecode_path=self.bytecode_path + ) + + def _generate_ets_stdlib(self, stdlib_path : str, output_path : str): + dummy_test=path.join(self.test_root, "stdlib/std/core/Boolean.ets") + options=f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" + cmd_string=f"{self.compiler} {options}" + args=shlex.split(cmd_string) + process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) - def _generate_ets_stdlib(): - pass + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Cannot generate ETS stdlib : timeout") - + if process.returncode != 0: + raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - \ No newline at end of file diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index 55b0dafbe..aba767ab7 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -1,3 +1,6 @@ +import os.path +import shlex +import subprocess from runner_base import correct_path, get_test_id from runner_ets import RunnerETS from test_ets_cts import TestETS_CTS @@ -6,9 +9,34 @@ from test_ets_cts import TestETS_CTS class RunnerETS_CTS(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-cts") + self.etc_cts_generated_code="/tmp/cts-generated" + self.code_generator="scripts/formatChecker/generate.py" + self.python_engine="python3" + self._generate_ets_code_from_template() + self.test_root=self.etc_cts_generated_code + self.add_directory(self.etc_cts_generated_code, "ets", []) + + + def _generate_ets_code_from_template(self): + cmd_string = f"{self.python_engine} {self.code_generator} ./CTS {self.etc_cts_generated_code}" + args=shlex.split(cmd_string) + work_dir = os.path.join(self.test_root, "tests") + process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_dir, env=self.cmd_env) + + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Cannot generate CTS code from template : timeout") + + if process.returncode != 0: + raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) + - def _generate_ets_code_from_template(): - pass + def create_test(self, test_file, flags, is_ignored): + test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) + test.ignored = is_ignored + return test diff --git a/test/runner/starter.py b/test/runner/starter.py index 553faa5f3..599ee95fd 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -44,7 +44,7 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - parser.add_argument('--ets-cts', action='store_true', dest='ets_sts', + parser.add_argument('--ets-cts', action='store_true', dest='ets_cts', default=False, help='run test agaisnt ETS specification') parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', default=False, help='run test agaisnt ETS specification') diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 8ec62c3d5..e75bf4680 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -10,8 +10,16 @@ from test_base import Test class TestETS(Test): - def __init__(self, test_env, test_path, flags, test_id, update_expected): + def __init__(self, test_path, test_env, flags, test_id, update_expected): Test.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further - self.fail_kind = None \ No newline at end of file + self.fail_kind = None + + def run_compiler(self): + options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --output={} --opt-level=0 {self.test_path}" + self.test_env.compiler + pass + + def run_runtime(): + pass \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py index 80e421fa8..8933ad052 100644 --- a/test/runner/test_ets_cts.py +++ b/test/runner/test_ets_cts.py @@ -3,9 +3,9 @@ from os import path, makedirs from configuration_kind import ConfigurationKind from test_ets import TestETS from utils import purify - +# test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) class TestETS_CTS(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + def __init__(self, test_path, test_env, flags, test_id=None, update_expected=False): TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) self.tmp_dir = path.join(path.sep, "tmp", "cts") makedirs(self.tmp_dir, exist_ok=True) -- Gitee From 516e23f05e1567133cec4c2c997c1bdf670b5e78 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Wed, 19 Oct 2022 16:16:50 +0300 Subject: [PATCH 17/36] execution of CTS was debugged --- test/runner/runner_base.py | 1 + test/runner/runner_ets.py | 55 +++++++++++++++++------------- test/runner/runner_ets_cts.py | 5 +-- test/runner/test_ets.py | 64 +++++++++++++++++++++++++++++++---- test/runner/test_ets_cts.py | 2 +- 5 files changed, 93 insertions(+), 34 deletions(-) diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 506e7c46a..0aadc492a 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -180,6 +180,7 @@ class Runner: ) ))) + def create_test(self, test_file, flags, is_ignored): pass diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 0f028d6e3..11f7ef6e7 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -28,40 +28,47 @@ INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" class RunnerETS(Runner): - def __init__(self, args, name:str): + def __init__(self, args, name: str): Runner.__init__(self, args, name) - output_path = path.join(self.stdlib_path, self.stdlib_name) + self.stdlib_path = path.join(self.test_root, "stdlib") + self.stdlib_name = "etsstdlib.abc" + stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) self.cmd_env = environ.copy() - self.test_root=args.test_root - self.compiler=path.join(self.build_dir, "bin", "es2panda") + self.test_root = args.test_root + self.compiler = path.join(self.build_dir, "bin", "es2panda") self.runtime = path.join(self.build_dir, "bin", "ark") - self.runtime_args = f"--boot-panda={output_path} --load_runtime=ets" self.cmd_env = environ.copy() - self.stdlib_name="etsstdlib.abc" - self.bytecode_path=path.join("/tmp", "ets", "cts") - self.stdlib_path=path.join(self.test_root, "stdlib") - self.compiler_args = f"--stdlib={self.stdlib_path} --gen-stdlib=false --extension=ets --opt_level=0" + self.stdlib_name = "etsstdlib.abc" + self.stdlib_path = path.join(self.test_root, "stdlib") - stdlib_file = Path(output_path) + + stdlib_file = Path(stdlib_output_path) if not stdlib_file.exists(): - self._generate_ets_stdlib(self.stdlib_path, output_path) + self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) self.test_env = TestEnv( args=args, cmd_env=self.cmd_env, - compiler=self.compiler, + es2panda =self.compiler, runtime=self.runtime, - compiler_args = self.compiler_args, - runtime_args=self.runtime_args, - stdlib_path=self.stdlib_path, - bytecode_path=self.bytecode_path + runtime_args="", + arkaout="", + aot_args="", + ark_quick="", + quick_args="", + conf_kind="", + cmd_prefix="" ) - def _generate_ets_stdlib(self, stdlib_path : str, output_path : str): - dummy_test=path.join(self.test_root, "stdlib/std/core/Boolean.ets") - options=f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" - cmd_string=f"{self.compiler} {options}" - args=shlex.split(cmd_string) - process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) + self.test_env.stdlib_path = self.stdlib_path + self.test_env.stdlib_output_path = stdlib_output_path + + def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): + dummy_test = path.join(self.test_root, "stdlib/std/core/Boolean.ets") + options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" + cmd_string = f"{self.compiler} {options}" + args = shlex.split(cmd_string) + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) try: out, err = process.communicate(timeout=600) @@ -70,5 +77,5 @@ class RunnerETS(Runner): raise Exception("Cannot generate ETS stdlib : timeout") if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - + raise Exception("Generating ETS stdlib fail %d" % + (process.returncode)) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index aba767ab7..0b782f0c3 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -3,7 +3,7 @@ import shlex import subprocess from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_cts import TestETS_CTS +from test_ets import TestETS class RunnerETS_CTS(RunnerETS): @@ -13,6 +13,7 @@ class RunnerETS_CTS(RunnerETS): self.code_generator="scripts/formatChecker/generate.py" self.python_engine="python3" self._generate_ets_code_from_template() + self.test_root_backup = self.test_root self.test_root=self.etc_cts_generated_code self.add_directory(self.etc_cts_generated_code, "ets", []) @@ -34,7 +35,7 @@ class RunnerETS_CTS(RunnerETS): def create_test(self, test_file, flags, is_ignored): - test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) + test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) test.ignored = is_ignored return test diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index e75bf4680..b2568ec91 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -2,6 +2,7 @@ import subprocess import sys from os import path, remove from typing import List +import shlex from configuration_kind import ConfigurationKind from fail_kind import FailKind @@ -10,16 +11,65 @@ from test_base import Test class TestETS(Test): - def __init__(self, test_path, test_env, flags, test_id, update_expected): + def __init__(self, test_env, test_path, flags, test_id, update_expected=False): Test.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None + self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" + self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" + self.common_ets_runtime_options = test_env.runtime_args + self.ets_main_entry_point="ETSGLOBAL::main" + self.bytecode_path = path.join("/tmp", "ets", "cts") - def run_compiler(self): - options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --output={} --opt-level=0 {self.test_path}" - self.test_env.compiler - pass + def do_run(self): + test_basename = path.basename(self.path) + is_negative_compilation = False + is_negative_runtime = False + if test_basename.startswith("n."): + is_negative_compilation = True + if test_basename.startswith("n.e."): + is_negative_runtime = True - def run_runtime(): - pass \ No newline at end of file + compiled_file, rc = self._run_compiler(is_negative_compilation) + if rc > 0 : + return ; + + if not is_negative_compilation: + self._run_runtime(compiled_file, is_negative_runtime) + + def _run_compiler(self, is_negative = False) -> str: + test_basename = path.basename(self.path) + test_abc = f"{test_basename}.abc" + output_path = path.join(self.bytecode_path, "abc", test_abc) + ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" + args = shlex.split(compiler_cmd_string) + rc = self._run_process_from_python(args, is_negative) + return (output_path,rc) + + + def _run_runtime(self, abc_file:str, is_negative:bool): + runtime_cmd_string = f"{self.common_ets_runtime_options} {abc_file} {self.ets_main_entry_point}" + args = shlex.split(runtime_cmd_string) + runtime_rc = self._run_process_from_python(args, is_negative) + return runtime_rc + + def _run_process_from_python(self, args:list, is_negative:bool)->None: + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) + + if process.returncode != 0: + if is_negative: + return 0 + else: + return 1 + + + + \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py index 8933ad052..ff1e7f510 100644 --- a/test/runner/test_ets_cts.py +++ b/test/runner/test_ets_cts.py @@ -5,7 +5,7 @@ from test_ets import TestETS from utils import purify # test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) class TestETS_CTS(TestETS): - def __init__(self, test_path, test_env, flags, test_id=None, update_expected=False): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) self.tmp_dir = path.join(path.sep, "tmp", "cts") makedirs(self.tmp_dir, exist_ok=True) -- Gitee From 93a1da5ae3459ac99ca76bcdfb6c0108a21a2bc6 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Wed, 19 Oct 2022 19:30:16 +0300 Subject: [PATCH 18/36] hot fixes for execution of CTS was debugged --- test/runner/test_ets.py | 52 +++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index b2568ec91..4c6353437 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -1,6 +1,6 @@ import subprocess import sys -from os import path, remove +from os import path, makedirs from typing import List import shlex @@ -18,9 +18,9 @@ class TestETS(Test): self.fail_kind = None self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" - self.common_ets_runtime_options = test_env.runtime_args self.ets_main_entry_point="ETSGLOBAL::main" - self.bytecode_path = path.join("/tmp", "ets", "cts") + self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") + makedirs(self.bytecode_path, exist_ok=True) def do_run(self): test_basename = path.basename(self.path) @@ -33,42 +33,54 @@ class TestETS(Test): compiled_file, rc = self._run_compiler(is_negative_compilation) if rc > 0 : - return ; + return self; + else: + print("Compilation Passed") if not is_negative_compilation: - self._run_runtime(compiled_file, is_negative_runtime) + runtime_rc = self._run_runtime(compiled_file, is_negative_runtime) + if runtime_rc == 0: + print ("Runtime Passed") + else: + print ("Runtime Failed") + return self - def _run_compiler(self, is_negative = False) -> str: + def _run_compiler(self, is_negative:bool) -> str: test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" - output_path = path.join(self.bytecode_path, "abc", test_abc) + output_path = path.join(self.bytecode_path, test_abc) ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - rc = self._run_process_from_python(args, is_negative) - return (output_path,rc) + compile_rc = self._run_process_from_python(args) + if is_negative and compile_rc != 0: + return (output_path, 0) + elif is_negative and compile_rc == 0: + return (output_path, 1) + else: + return(output_path, 0) def _run_runtime(self, abc_file:str, is_negative:bool): - runtime_cmd_string = f"{self.common_ets_runtime_options} {abc_file} {self.ets_main_entry_point}" + runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - runtime_rc = self._run_process_from_python(args, is_negative) - return runtime_rc + runtime_rc = self._run_process_from_python(args) + if is_negative and runtime_rc != 0: + return 0 + elif is_negative and runtime_rc == 0: + return 1 + else: + return 0 - def _run_process_from_python(self, args:list, is_negative:bool)->None: + def _run_process_from_python(self, args:list): process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - out, err = process.communicate(timeout=600) + out, err = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - if process.returncode != 0: - if is_negative: - return 0 - else: - return 1 + return process.returncode != 0 -- Gitee From f1cfc1e0a2eda4a8c674f1737f7e2fd8097e70bc Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 11:27:12 +0300 Subject: [PATCH 19/36] update for report generating --- test/runner/test_ets.py | 45 ++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 4c6353437..2a3a52bdf 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -31,18 +31,13 @@ class TestETS(Test): if test_basename.startswith("n.e."): is_negative_runtime = True - compiled_file, rc = self._run_compiler(is_negative_compilation) - if rc > 0 : + self.compile_report, compiled_file = self._run_compiler(is_negative_compilation) + if self.compile_report.return_code > 0 : return self; - else: - print("Compilation Passed") if not is_negative_compilation: - runtime_rc = self._run_runtime(compiled_file, is_negative_runtime) - if runtime_rc == 0: - print ("Runtime Passed") - else: - print ("Runtime Failed") + self.runtime_report = self._run_runtime(compiled_file, is_negative_runtime) + return self def _run_compiler(self, is_negative:bool) -> str: @@ -52,35 +47,47 @@ class TestETS(Test): ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - compile_rc = self._run_process_from_python(args) + output_content, error_content, compile_rc = self._run_process_from_python(args) + compile_report = TestReport( + output=output_content, + error = error_content, + return_code = 0 + ) if is_negative and compile_rc != 0: - return (output_path, 0) + return (compile_report, output_path) elif is_negative and compile_rc == 0: - return (output_path, 1) + compile_report.return_code = 1 + return (compile_report, output_path) else: - return(output_path, 0) + return(compile_report, output_path) def _run_runtime(self, abc_file:str, is_negative:bool): runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - runtime_rc = self._run_process_from_python(args) + output_content, error_content, runtime_rc = self._run_process_from_python(args) + runtime_report = TestReport( + output=output_content, + error = error_content, + return_code = 0 + ) if is_negative and runtime_rc != 0: - return 0 + return runtime_report elif is_negative and runtime_rc == 0: - return 1 + runtime_report.return_code = 1 + return runtime_report else: - return 0 + return runtime_report def _run_process_from_python(self, args:list): process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: - out, err = process.communicate(timeout=60) + output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - return process.returncode != 0 + return (output_content, err_content,process.returncode) -- Gitee From 18c30768005267deb8e57f1a394141fca4641ec1 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 12:44:48 +0300 Subject: [PATCH 20/36] report generating was debugged --- test/runner/runner_ets.py | 5 +- test/runner/runner_ets_stdlib.py | 5 +- .../{runner_js.py => runner_file_based.py} | 2 +- test/runner/runner_js_hermes.py | 6 +- test/runner/runner_js_parser.py | 4 +- test/runner/runner_js_test262.py | 6 +- test/runner/test_ets.py | 80 +++++++++++-------- test/runner/test_ets_cts.py | 13 --- test/runner/test_ets_stdlib.py | 13 --- 9 files changed, 61 insertions(+), 73 deletions(-) rename test/runner/{runner_js.py => runner_file_based.py} (99%) delete mode 100644 test/runner/test_ets_cts.py delete mode 100644 test/runner/test_ets_stdlib.py diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 11f7ef6e7..f0720a5f9 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -13,6 +13,7 @@ from fail_kind import FailKind from params import TestEnv from report_format import ReportFormat from runner_base import Runner +from runner_file_based import RunnerFileBased from utils import write_2_file INDEX_TITLE = "${Title}" @@ -27,9 +28,9 @@ INDEX_TEST_NAME = "${TestName}" INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" -class RunnerETS(Runner): +class RunnerETS(RunnerFileBased): def __init__(self, args, name: str): - Runner.__init__(self, args, name) + RunnerFileBased.__init__(self, args, name) self.stdlib_path = path.join(self.test_root, "stdlib") self.stdlib_name = "etsstdlib.abc" stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index cf801d317..c8ebbd373 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,15 +1,12 @@ from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_stdlib import TestETS_STDLIB class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): - RunnerETS.__init__(self, args, "ets-cts") - - def _generate_ets_code_from_template(): pass + \ No newline at end of file diff --git a/test/runner/runner_js.py b/test/runner/runner_file_based.py similarity index 99% rename from test/runner/runner_js.py rename to test/runner/runner_file_based.py index 6d18d102f..410d6faf9 100644 --- a/test/runner/runner_js.py +++ b/test/runner/runner_file_based.py @@ -24,7 +24,7 @@ INDEX_TEST_NAME = "${TestName}" INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" -class RunnerJS(Runner): +class RunnerFileBased(Runner): def __init__(self, args, name): Runner.__init__(self, args, name) self.cmd_env = environ.copy() diff --git a/test/runner/runner_js_hermes.py b/test/runner/runner_js_hermes.py index 79cf36021..9c20648b9 100644 --- a/test/runner/runner_js_hermes.py +++ b/test/runner/runner_js_hermes.py @@ -1,12 +1,12 @@ from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_hermes import TestJSHermes from util_hermes import UtilHermes -class RunnerJSHermes(RunnerJS): +class RunnerJSHermes(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "hermes") + RunnerFileBased.__init__(self, args, "hermes") self.collect_excluded_test_lists() self.collect_ignored_test_lists() diff --git a/test/runner/runner_js_parser.py b/test/runner/runner_js_parser.py index 2a134867c..d10fe8bef 100644 --- a/test/runner/runner_js_parser.py +++ b/test/runner/runner_js_parser.py @@ -1,11 +1,11 @@ from os import path from runner_base import get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_parser import TestJSParser -class RunnerJSParser(RunnerJS): +class RunnerJSParser(RunnerFileBased): def __init__(self, args): super(RunnerJSParser, self).__init__(args, "parser-js") diff --git a/test/runner/runner_js_test262.py b/test/runner/runner_js_test262.py index 21a880723..d16af1f07 100644 --- a/test/runner/runner_js_test262.py +++ b/test/runner/runner_js_test262.py @@ -1,14 +1,14 @@ from os import path from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_test262 import TestJSTest262 from util_test262 import UtilTest262 -class RunnerJSTest262(RunnerJS): +class RunnerJSTest262(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "test262-ark") + RunnerFileBased.__init__(self, args, "test262-ark") self.ignored_name_prefix = "test262" self.collect_excluded_test_lists(test_name=self.ignored_name_prefix) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 2a3a52bdf..22aee1fa9 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -12,13 +12,14 @@ from test_base import Test class TestETS(Test): def __init__(self, test_env, test_path, flags, test_id, update_expected=False): - Test.__init__(self, test_env, test_path, flags, test_id, update_expected) + Test.__init__(self, test_env, test_path, + flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" - self.ets_main_entry_point="ETSGLOBAL::main" + self.ets_main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") makedirs(self.bytecode_path, exist_ok=True) @@ -31,45 +32,62 @@ class TestETS(Test): if test_basename.startswith("n.e."): is_negative_runtime = True - self.compile_report, compiled_file = self._run_compiler(is_negative_compilation) - if self.compile_report.return_code > 0 : - return self; + self.report, compiled_file = self._run_compiler(is_negative_compilation) + if self.report.return_code > 0: + self.passed=False + self.fail_kind= FailKind.ES2PANDA_FAIL + return self + elif self.report.return_code == 0 and is_negative_compilation: + self.passed = True + return self + else: + pass - if not is_negative_compilation: - self.runtime_report = self._run_runtime(compiled_file, is_negative_runtime) + if not is_negative_compilation: + self.report = self._run_runtime(compiled_file, is_negative_runtime) + if self.report.return_code > 0: + self.passed = False + self.fail_kind = FailKind.RUNTIME_FAIL + else: + self.passed = True return self - def _run_compiler(self, is_negative:bool) -> str: + def _run_compiler(self, is_negative: bool) -> str: test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - output_content, error_content, compile_rc = self._run_process_from_python(args) - compile_report = TestReport( - output=output_content, - error = error_content, - return_code = 0 + output_content, error_content, compile_rc = self._run_process_from_python( + args) + compile_report_succes = TestReport( + output=output_content.decode('UTF-8'), + error=error_content.decode('UTF-8'), + return_code=0 + ) + compile_report_fail = TestReport( + output=output_content.decode('UTF-8'), + error=error_content.decode('UTF-8'), + return_code=1 ) if is_negative and compile_rc != 0: - return (compile_report, output_path) + return (compile_report_succes, output_path) elif is_negative and compile_rc == 0: - compile_report.return_code = 1 - return (compile_report, output_path) + return (compile_report_fail, output_path) else: - return(compile_report, output_path) - + return (compile_report_succes, output_path) - def _run_runtime(self, abc_file:str, is_negative:bool): - runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" + def _run_runtime(self, abc_file: str, is_negative: bool): + runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - output_content, error_content, runtime_rc = self._run_process_from_python(args) + output_content, error_content, runtime_rc = self._run_process_from_python( + args) runtime_report = TestReport( output=output_content, - error = error_content, - return_code = 0 + error=error_content, + return_code=0 ) if is_negative and runtime_rc != 0: return runtime_report @@ -79,16 +97,14 @@ class TestETS(Test): else: return runtime_report - def _run_process_from_python(self, args:list): - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + def _run_process_from_python(self, args: list): + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() - raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - - return (output_content, err_content,process.returncode) - - + raise Exception( + "Hmm .... Task completed %s : timeout" % (self.test_id)) - \ No newline at end of file + return (output_content, err_content, process.returncode) diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py deleted file mode 100644 index ff1e7f510..000000000 --- a/test/runner/test_ets_cts.py +++ /dev/null @@ -1,13 +0,0 @@ -from os import path, makedirs - -from configuration_kind import ConfigurationKind -from test_ets import TestETS -from utils import purify -# test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) -class TestETS_CTS(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): - TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) - self.tmp_dir = path.join(path.sep, "tmp", "cts") - makedirs(self.tmp_dir, exist_ok=True) - self.util = self.test_env.util - diff --git a/test/runner/test_ets_stdlib.py b/test/runner/test_ets_stdlib.py deleted file mode 100644 index dd1b3fb07..000000000 --- a/test/runner/test_ets_stdlib.py +++ /dev/null @@ -1,13 +0,0 @@ -from os import path, makedirs - -from configuration_kind import ConfigurationKind -from test_ets import TestETS -from utils import purify - -class TestETS_STDLIB(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): - TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) - self.tmp_dir = path.join(path.sep, "tmp", "cts") - makedirs(self.tmp_dir, exist_ok=True) - self.util = self.test_env.util - -- Gitee From 1200c6e9c59d570eaabffe3eaa4acaee499602f5 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 18:09:18 +0300 Subject: [PATCH 21/36] update for stdlib --- test/runner/runner.py | 4 -- test/runner/runner_ets_cts.py | 44 -------------- test/runner/runner_ets_stdlib.py | 15 ++++- test/runner/test_ets.py | 98 +++++++++++++++----------------- 4 files changed, 58 insertions(+), 103 deletions(-) delete mode 100644 test/runner/runner_ets_cts.py diff --git a/test/runner/runner.py b/test/runner/runner.py index 74ca8960a..995ced502 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -4,7 +4,6 @@ from dotenv import load_dotenv from runner_js_hermes import RunnerJSHermes from runner_js_parser import RunnerJSParser from runner_js_test262 import RunnerJSTest262 -from runner_ets_cts import RunnerETS_CTS from runner_ets_stdlib import RunnerETS_STDLIB from starter import get_args @@ -29,9 +28,6 @@ def main(): if args.hermes: runners.append(RunnerJSHermes(args)) - if args.ets_cts: - runners.append(RunnerETS_CTS(args)) - if args.ets_stdlib: runners.append(RunnerETS_STDLIB(args)) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py deleted file mode 100644 index 0b782f0c3..000000000 --- a/test/runner/runner_ets_cts.py +++ /dev/null @@ -1,44 +0,0 @@ -import os.path -import shlex -import subprocess -from runner_base import correct_path, get_test_id -from runner_ets import RunnerETS -from test_ets import TestETS - - -class RunnerETS_CTS(RunnerETS): - def __init__(self, args): - RunnerETS.__init__(self, args, "ets-cts") - self.etc_cts_generated_code="/tmp/cts-generated" - self.code_generator="scripts/formatChecker/generate.py" - self.python_engine="python3" - self._generate_ets_code_from_template() - self.test_root_backup = self.test_root - self.test_root=self.etc_cts_generated_code - self.add_directory(self.etc_cts_generated_code, "ets", []) - - - def _generate_ets_code_from_template(self): - cmd_string = f"{self.python_engine} {self.code_generator} ./CTS {self.etc_cts_generated_code}" - args=shlex.split(cmd_string) - work_dir = os.path.join(self.test_root, "tests") - process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_dir, env=self.cmd_env) - - try: - out, err = process.communicate(timeout=600) - except subprocess.TimeoutExpired: - process.kill() - raise Exception("Cannot generate CTS code from template : timeout") - - if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - - - def create_test(self, test_file, flags, is_ignored): - test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) - test.ignored = is_ignored - return test - - - - \ No newline at end of file diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index c8ebbd373..f2058b6ed 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,10 +1,21 @@ -from runner_base import correct_path, get_test_id +import shlex +import os +import subprocess from runner_ets import RunnerETS +from test_ets import TestETS +from runner_base import get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): - pass + RunnerETS.__init__(self, args, "ets-stdlib") + self.add_directory(self.root_test, "ets", []) + + + def create_test(self, test_file, flags, is_ignored): + test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) + test.ignored = is_ignored + return test diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 22aee1fa9..3362b6d8e 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -2,6 +2,7 @@ import subprocess import sys from os import path, makedirs from typing import List +from typing import Tuple import shlex from configuration_kind import ConfigurationKind @@ -20,82 +21,74 @@ class TestETS(Test): self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" self.ets_main_entry_point = "ETSGLOBAL::main" - self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") + self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) def do_run(self): test_basename = path.basename(self.path) - is_negative_compilation = False - is_negative_runtime = False + is_negative_test = False if test_basename.startswith("n."): - is_negative_compilation = True - if test_basename.startswith("n.e."): - is_negative_runtime = True + is_negative_test = True - self.report, compiled_file = self._run_compiler(is_negative_compilation) - if self.report.return_code > 0: + report, compiled_file, fail_kind = self._run_compiler() + if report.return_code > 0: self.passed=False - self.fail_kind= FailKind.ES2PANDA_FAIL + self.fail_kind= fail_kind + self.report = report return self - elif self.report.return_code == 0 and is_negative_compilation: + + self.report = self._run_runtime(compiled_file, is_negative_test) + if self.report.return_code > 0 and is_negative_test: self.passed = True return self + elif self.report.return_code > 0: + self.passed = False + self.fail_kind = FailKind.RUNTIME_FAIL + return self else: - pass - - if not is_negative_compilation: - self.report = self._run_runtime(compiled_file, is_negative_runtime) - if self.report.return_code > 0: - self.passed = False - self.fail_kind = FailKind.RUNTIME_FAIL - else: - self.passed = True + self.passed = True + return self - return self - def _run_compiler(self, is_negative: bool) -> str: + def _run_compiler(self) -> Tuple[TestReport,str,FailKind]: + fail_kind = None test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - output_content, error_content, compile_rc = self._run_process_from_python( - args) - compile_report_succes = TestReport( - output=output_content.decode('UTF-8'), - error=error_content.decode('UTF-8'), - return_code=0 - ) - compile_report_fail = TestReport( - output=output_content.decode('UTF-8'), - error=error_content.decode('UTF-8'), - return_code=1 + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + try: + stdout_content, stderr_content = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + fail_kind = FailKind.ES2PANDA_TIMEOUT + + report = TestReport( + output=stdout_content, + error=stderr_content, + return_code=process.returncode ) - if is_negative and compile_rc != 0: - return (compile_report_succes, output_path) - elif is_negative and compile_rc == 0: - return (compile_report_fail, output_path) - else: - return (compile_report_succes, output_path) + return (report, output_path, fail_kind) def _run_runtime(self, abc_file: str, is_negative: bool): + fail_kind = None runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - output_content, error_content, runtime_rc = self._run_process_from_python( - args) + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + try: + stdout_content, stderr_content = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + fail_kind = FailKind.RUNTIME_TIMEOUT + runtime_report = TestReport( - output=output_content, - error=error_content, - return_code=0 + output=stdout_content, + error=stderr_content, + return_code=process.returncode ) - if is_negative and runtime_rc != 0: - return runtime_report - elif is_negative and runtime_rc == 0: - runtime_report.return_code = 1 - return runtime_report - else: - return runtime_report + return runtime_report, fail_kind def _run_process_from_python(self, args: list): process = subprocess.Popen( @@ -104,7 +97,6 @@ class TestETS(Test): output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() - raise Exception( - "Hmm .... Task completed %s : timeout" % (self.test_id)) + fail_kind = FailKind.PROCESS_TIMEOUT - return (output_content, err_content, process.returncode) + return (output_content, err_content, process.returncode, fail_kind) -- Gitee From 23a140aa4bd44cc3e391266c705af97da7a4be9a Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 18:24:27 +0300 Subject: [PATCH 22/36] update for ets-stdlib.not debugged --- test/runner/runner_ets.py | 4 +++- test/runner/starter.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index f0720a5f9..68c37d7b9 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -64,7 +64,7 @@ class RunnerETS(RunnerFileBased): self.test_env.stdlib_output_path = stdlib_output_path def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): - dummy_test = path.join(self.test_root, "stdlib/std/core/Boolean.ets") + dummy_test = path.join(self.test_root, "std/dummy.ets") options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" cmd_string = f"{self.compiler} {options}" args = shlex.split(cmd_string) @@ -77,6 +77,8 @@ class RunnerETS(RunnerFileBased): process.kill() raise Exception("Cannot generate ETS stdlib : timeout") + print("StdOut : " + out.decode('UTF-8')) + print("StdErr : " + err.decode('UTF-8')) if process.returncode != 0: raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) diff --git a/test/runner/starter.py b/test/runner/starter.py index 599ee95fd..5853fc662 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -44,8 +44,6 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - parser.add_argument('--ets-cts', action='store_true', dest='ets_cts', - default=False, help='run test agaisnt ETS specification') parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', default=False, help='run test agaisnt ETS specification') parser.add_argument( -- Gitee From 0c5274af6106aba50047442fac6224ea2473412f Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Fri, 21 Oct 2022 16:05:59 +0300 Subject: [PATCH 23/36] support for stdlib test is ready for review 1 --- test/runner/runner_ets.py | 57 +++++++++++++++----------------- test/runner/runner_ets_stdlib.py | 3 +- test/runner/test_ets.py | 52 +++++++++++++---------------- 3 files changed, 51 insertions(+), 61 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 68c37d7b9..68afaa391 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -1,11 +1,6 @@ -import subprocess -import sys -from datetime import datetime -from glob import glob -from os import path, environ, makedirs -from typing import List +from cProfile import run +from os import path, environ from pathlib import Path -import shlex from configuration_kind import ConfigurationKind @@ -39,19 +34,21 @@ class RunnerETS(RunnerFileBased): self.compiler = path.join(self.build_dir, "bin", "es2panda") self.runtime = path.join(self.build_dir, "bin", "ark") self.cmd_env = environ.copy() - self.stdlib_name = "etsstdlib.abc" - self.stdlib_path = path.join(self.test_root, "stdlib") + self.stdlib_path = path.join(self.build_dir, "etsstdlib.abc") + self.stdlib_src_path = path.join(self.test_root, "stdlib") + self._check_binary_artefacts() + stdlib_file = Path(stdlib_output_path) - if not stdlib_file.exists(): - self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) + # if not stdlib_file.exists(): + # self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) self.test_env = TestEnv( args=args, cmd_env=self.cmd_env, es2panda =self.compiler, runtime=self.runtime, - runtime_args="", + runtime_args=f"--boot-panda-files={self.stdlib_path} --load-runtimes=ets", arkaout="", aot_args="", ark_quick="", @@ -60,25 +57,23 @@ class RunnerETS(RunnerFileBased): cmd_prefix="" ) - self.test_env.stdlib_path = self.stdlib_path - self.test_env.stdlib_output_path = stdlib_output_path + self.test_env.es2panda_args= f"--stdlib={self.stdlib_src_path} --gen-stdlib=false --extension=ets --opt-level=0" + + + def _check_binary_artefacts(self) : + compiler_path_obj = Path(self.compiler) + runtime_path_obj = Path(self.runtime) + stdlib_path_obj = Path(self.stdlib_path) + stdlib_src_path_obj = Path(self.stdlib_src_path) + + if not compiler_path_obj.is_file(): + raise Exception("Hmm ... ETS Compiler was not found") - def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): - dummy_test = path.join(self.test_root, "std/dummy.ets") - options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" - cmd_string = f"{self.compiler} {options}" - args = shlex.split(cmd_string) - process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) + if not runtime_path_obj.is_file(): + raise Exception("Hmm ... ETS Runtime was not found") - try: - out, err = process.communicate(timeout=600) - except subprocess.TimeoutExpired: - process.kill() - raise Exception("Cannot generate ETS stdlib : timeout") + if not stdlib_path_obj.is_file(): + raise Exception("Hmm ... standart library abc file not found") - print("StdOut : " + out.decode('UTF-8')) - print("StdErr : " + err.decode('UTF-8')) - if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % - (process.returncode)) + if not stdlib_src_path_obj.is_dir(): + raise Exception("Hmm ... Source code of standart library was not found") \ No newline at end of file diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index f2058b6ed..6187e4782 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -9,7 +9,8 @@ from runner_base import get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-stdlib") - self.add_directory(self.root_test, "ets", []) + self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") + self.add_directory(self.stdlib_test_path, "ets", []) def create_test(self, test_file, flags, is_ignored): diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 3362b6d8e..172d4f1d4 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -18,8 +18,6 @@ class TestETS(Test): # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None - self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" - self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" self.ets_main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) @@ -31,19 +29,18 @@ class TestETS(Test): is_negative_test = True report, compiled_file, fail_kind = self._run_compiler() - if report.return_code > 0: - self.passed=False - self.fail_kind= fail_kind - self.report = report + self.fail_kind= fail_kind + self.report = report + if report.return_code != 0: + self.passed = False return self - self.report = self._run_runtime(compiled_file, is_negative_test) - if self.report.return_code > 0 and is_negative_test: + self.report, self.fail_kind = self._run_runtime(compiled_file) + if self.report.return_code != 0 and is_negative_test: self.passed = True return self - elif self.report.return_code > 0: + elif self.report.return_code != 0: self.passed = False - self.fail_kind = FailKind.RUNTIME_FAIL return self else: self.passed = True @@ -55,9 +52,10 @@ class TestETS(Test): test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + ets_compiler_options = f"{self.test_env.es2panda_args} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) + self.log_cmd(args) process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: stdout_content, stderr_content = process.communicate(timeout=60) @@ -65,17 +63,21 @@ class TestETS(Test): process.kill() fail_kind = FailKind.ES2PANDA_TIMEOUT + if process.returncode != 0: + fail_kind = FailKind.ES2PANDA_FAIL + report = TestReport( - output=stdout_content, - error=stderr_content, + output=stdout_content.decode('UTF-8'), + error=stderr_content.decode('UTF-8'), return_code=process.returncode ) return (report, output_path, fail_kind) - def _run_runtime(self, abc_file: str, is_negative: bool): + def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: fail_kind = None - runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" + runtime_cmd_string = f"{self.test_env.runtime} {self.test_env.runtime_args} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) + self.log_cmd(args) process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: stdout_content, stderr_content = process.communicate(timeout=60) @@ -83,20 +85,12 @@ class TestETS(Test): process.kill() fail_kind = FailKind.RUNTIME_TIMEOUT + if process.returncode != 0: + fail_kind = FailKind.RUNTIME_FAIL + runtime_report = TestReport( - output=stdout_content, - error=stderr_content, + output=stdout_content.decode('UTF-8'), + error=stderr_content.decode('UTF-8'), return_code=process.returncode ) - return runtime_report, fail_kind - - def _run_process_from_python(self, args: list): - process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - output_content, err_content = process.communicate(timeout=60) - except subprocess.TimeoutExpired: - process.kill() - fail_kind = FailKind.PROCESS_TIMEOUT - - return (output_content, err_content, process.returncode, fail_kind) + return runtime_report, fail_kind \ No newline at end of file -- Gitee From a30bd98ecc1d1b4818b93aa6a87a1ff562b7d1e9 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 14:53:09 +0300 Subject: [PATCH 24/36] add excluded list processing --- test/ets-stdlib-excluded.txt | 3 +++ test/runner/runner_base.py | 1 + test/runner/runner_ets_stdlib.py | 5 ++++- test/runner/test_ets.py | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 test/ets-stdlib-excluded.txt diff --git a/test/ets-stdlib-excluded.txt b/test/ets-stdlib-excluded.txt new file mode 100644 index 000000000..477b26172 --- /dev/null +++ b/test/ets-stdlib-excluded.txt @@ -0,0 +1,3 @@ +tests/stdlib/std/math/sqrt-negative-01.ets +tests/stdlib/std/math/sqrt-positive-03.ets +tests/stdlib/std/math/sqrt-positive-04.ets \ No newline at end of file diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 0aadc492a..a1c6fbbd2 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -133,6 +133,7 @@ class Runner: # Read excluded_lists and load list of excluded tests def load_excluded_tests(self): + print(self.excluded_lists) self.excluded_tests.update(self.load_tests_from_lists(self.excluded_lists)) self.excluded = len(self.excluded_tests) diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index 6187e4782..97dd11067 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -3,14 +3,17 @@ import os import subprocess from runner_ets import RunnerETS from test_ets import TestETS -from runner_base import get_test_id +from runner_base import correct_path, get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-stdlib") self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") + self.ignored_name_prefix = self.name + self.excluded_lists = [correct_path(self.list_root, f"{self.ignored_name_prefix}-excluded.txt")] self.add_directory(self.stdlib_test_path, "ets", []) + print (self.excluded_lists) def create_test(self, test_file, flags, is_ignored): diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 172d4f1d4..e7d666207 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -71,6 +71,7 @@ class TestETS(Test): error=stderr_content.decode('UTF-8'), return_code=process.returncode ) + return (report, output_path, fail_kind) def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: -- Gitee From 081697caa26c7c1f38360ab012959d975fb2461e Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 16:06:01 +0300 Subject: [PATCH 25/36] remove shlex module usage --- test/runner/runner_ets.py | 7 +++++-- test/runner/test_ets.py | 25 +++++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 68afaa391..6575b5347 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -48,7 +48,7 @@ class RunnerETS(RunnerFileBased): cmd_env=self.cmd_env, es2panda =self.compiler, runtime=self.runtime, - runtime_args=f"--boot-panda-files={self.stdlib_path} --load-runtimes=ets", + runtime_args=[f"--boot-panda-files={self.stdlib_path}", "--load-runtimes=ets"], arkaout="", aot_args="", ark_quick="", @@ -57,7 +57,10 @@ class RunnerETS(RunnerFileBased): cmd_prefix="" ) - self.test_env.es2panda_args= f"--stdlib={self.stdlib_src_path} --gen-stdlib=false --extension=ets --opt-level=0" + self.test_env.es2panda_args= [f"--stdlib={self.stdlib_src_path}", + "--gen-stdlib=false", + "--extension=ets", + "--opt-level=0"] def _check_binary_artefacts(self) : diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index e7d666207..4a906e69f 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -1,9 +1,9 @@ +from cProfile import run import subprocess import sys from os import path, makedirs from typing import List from typing import Tuple -import shlex from configuration_kind import ConfigurationKind from fail_kind import FailKind @@ -52,11 +52,13 @@ class TestETS(Test): test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - ets_compiler_options = f"{self.test_env.es2panda_args} --output={output_path} {self.path}" - compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" - args = shlex.split(compiler_cmd_string) - self.log_cmd(args) - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + compiler_cmd_string = [] + compiler_cmd_string.append(self.test_env.es2panda) + compiler_cmd_string.extend(self.test_env.es2panda_args) + compiler_cmd_string.append(f"--output={output_path}") + compiler_cmd_string.append(self.path) + self.log_cmd(compiler_cmd_string) + process = subprocess.Popen(compiler_cmd_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: stdout_content, stderr_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: @@ -76,10 +78,13 @@ class TestETS(Test): def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: fail_kind = None - runtime_cmd_string = f"{self.test_env.runtime} {self.test_env.runtime_args} {abc_file} {self.ets_main_entry_point}" - args = shlex.split(runtime_cmd_string) - self.log_cmd(args) - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + runtime_cmd_string = [] + runtime_cmd_string.append(self.test_env.runtime) + runtime_cmd_string.extend(self.test_env.runtime_args) + runtime_cmd_string.append(abc_file) + runtime_cmd_string.append(self.ets_main_entry_point) + self.log_cmd(runtime_cmd_string) + process = subprocess.Popen(runtime_cmd_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: stdout_content, stderr_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: -- Gitee From 07c106fa9d8b0f5d3b5f19cca5118db9cdf5f672 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 16:22:08 +0300 Subject: [PATCH 26/36] remove debug print --- test/runner/runner_base.py | 1 - test/runner/runner_ets_stdlib.py | 1 - 2 files changed, 2 deletions(-) diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index a1c6fbbd2..0aadc492a 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -133,7 +133,6 @@ class Runner: # Read excluded_lists and load list of excluded tests def load_excluded_tests(self): - print(self.excluded_lists) self.excluded_tests.update(self.load_tests_from_lists(self.excluded_lists)) self.excluded = len(self.excluded_tests) diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index 97dd11067..4c384d6e1 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -13,7 +13,6 @@ class RunnerETS_STDLIB(RunnerETS): self.ignored_name_prefix = self.name self.excluded_lists = [correct_path(self.list_root, f"{self.ignored_name_prefix}-excluded.txt")] self.add_directory(self.stdlib_test_path, "ets", []) - print (self.excluded_lists) def create_test(self, test_file, flags, is_ignored): -- Gitee From c05c460f8beda61412bceb37f50aa4d8e2a626c0 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 16:26:26 +0300 Subject: [PATCH 27/36] replace exception with file not found exception --- test/runner/runner_ets.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index f89444187..7ad533b1c 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -59,13 +59,13 @@ class RunnerETS(RunnerFileBased): stdlib_src_path_obj = Path(self.stdlib_src_path) if not compiler_path_obj.is_file(): - raise Exception("Hmm ... ETS Compiler was not found") + raise FileNotFoundError("Hmm ... ETS Compiler was not found") if not runtime_path_obj.is_file(): - raise Exception("Hmm ... ETS Runtime was not found") + raise FileNotFoundError("Hmm ... ETS Runtime was not found") if not stdlib_path_obj.is_file(): - raise Exception("Hmm ... standart library abc file not found") + raise FileNotFoundError("Hmm ... standart library abc file not found") if not stdlib_src_path_obj.is_dir(): - raise Exception("Hmm ... Source code of standart library was not found") \ No newline at end of file + raise FileNotFoundError("Hmm ... Source code of standart library was not found") \ No newline at end of file -- Gitee From 8751560380b830c83b0679ba3022e35d36010439 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 16:29:45 +0300 Subject: [PATCH 28/36] replace empty string by list --- test/runner/runner_ets.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 7ad533b1c..61ca46b12 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -38,12 +38,12 @@ class RunnerETS(RunnerFileBased): es2panda =self.compiler, runtime=self.runtime, runtime_args=[f"--boot-panda-files={self.stdlib_path}", "--load-runtimes=ets"], - arkaout="", - aot_args="", - ark_quick="", - quick_args="", - conf_kind="", - cmd_prefix="" + arkaout=[], + aot_args=[], + ark_quick=[], + quick_args=[], + conf_kind=[], + cmd_prefix=[] ) self.test_env.es2panda_args= [f"--stdlib={self.stdlib_src_path}", -- Gitee From 15bcadd958d48a14484db204a0dd8f2aecc0d129 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 17:00:52 +0300 Subject: [PATCH 29/36] fix review issues by Victoria --- test/runner/runner_ets.py | 13 +------------ test/runner/runner_ets_stdlib.py | 2 -- test/runner/starter.py | 2 +- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 61ca46b12..c21c682e9 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -1,13 +1,8 @@ -from cProfile import run from os import path, environ from pathlib import Path -from configuration_kind import ConfigurationKind -from fail_kind import FailKind from params import TestEnv -from report_format import ReportFormat -from runner_base import Runner from runner_file_based import RunnerFileBased from utils import write_2_file @@ -17,7 +12,6 @@ class RunnerETS(RunnerFileBased): RunnerFileBased.__init__(self, args, name) self.stdlib_path = path.join(self.test_root, "stdlib") self.stdlib_name = "etsstdlib.abc" - stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) self.cmd_env = environ.copy() self.test_root = args.test_root self.compiler = path.join(self.build_dir, "bin", "es2panda") @@ -26,12 +20,7 @@ class RunnerETS(RunnerFileBased): self.stdlib_path = path.join(self.build_dir, "etsstdlib.abc") self.stdlib_src_path = path.join(self.test_root, "stdlib") self._check_binary_artefacts() - - - stdlib_file = Path(stdlib_output_path) - # if not stdlib_file.exists(): - # self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) self.test_env = TestEnv( args=args, cmd_env=self.cmd_env, @@ -65,7 +54,7 @@ class RunnerETS(RunnerFileBased): raise FileNotFoundError("Hmm ... ETS Runtime was not found") if not stdlib_path_obj.is_file(): - raise FileNotFoundError("Hmm ... standart library abc file not found") + raise FileNotFoundError("Hmm ... standard library abc file not found") if not stdlib_src_path_obj.is_dir(): raise FileNotFoundError("Hmm ... Source code of standart library was not found") \ No newline at end of file diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index 4c384d6e1..1f0fcab74 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,6 +1,4 @@ -import shlex import os -import subprocess from runner_ets import RunnerETS from test_ets import TestETS from runner_base import correct_path, get_test_id diff --git a/test/runner/starter.py b/test/runner/starter.py index 5853fc662..62e2eb17e 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -45,7 +45,7 @@ def get_args(): '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', - default=False, help='run test agaisnt ETS specification') + default=False, help='run test against ETS specification') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), help='directory with test file. If not set the module directory is used') -- Gitee From 87d33262deea3245d5eaa8be72f923473ba8c711 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 17:26:32 +0300 Subject: [PATCH 30/36] fix review issues --- test/runner/params.py | 1 + test/runner/runner_ets.py | 10 ++++------ test/runner/runner_file_based.py | 1 + test/runner/test_ets.py | 16 ++++------------ 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/test/runner/params.py b/test/runner/params.py index df994ed45..ce108b22e 100644 --- a/test/runner/params.py +++ b/test/runner/params.py @@ -15,6 +15,7 @@ class TestEnv: cmd_env: Any es2panda: str + es2panda_args : List[str] runtime: str runtime_args: List[str] diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index c21c682e9..24cbb746c 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -25,6 +25,10 @@ class RunnerETS(RunnerFileBased): args=args, cmd_env=self.cmd_env, es2panda =self.compiler, + es2panda_args= [f"--stdlib={self.stdlib_src_path}", + "--gen-stdlib=false", + "--extension=ets", + "--opt-level=0"], runtime=self.runtime, runtime_args=[f"--boot-panda-files={self.stdlib_path}", "--load-runtimes=ets"], arkaout=[], @@ -35,12 +39,6 @@ class RunnerETS(RunnerFileBased): cmd_prefix=[] ) - self.test_env.es2panda_args= [f"--stdlib={self.stdlib_src_path}", - "--gen-stdlib=false", - "--extension=ets", - "--opt-level=0"] - - def _check_binary_artefacts(self) : compiler_path_obj = Path(self.compiler) runtime_path_obj = Path(self.runtime) diff --git a/test/runner/runner_file_based.py b/test/runner/runner_file_based.py index 410d6faf9..715219d48 100644 --- a/test/runner/runner_file_based.py +++ b/test/runner/runner_file_based.py @@ -113,6 +113,7 @@ class RunnerFileBased(Runner): cmd_prefix=self.cmd_prefix, cmd_env=self.cmd_env, es2panda=self.es2panda, + es2panda_args=[], runtime=self.runtime, runtime_args=self.runtime_args, arkaout=self.arkaot, diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 66d137b4b..61ad11381 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -1,12 +1,9 @@ import subprocess -import sys from os import path, makedirs -from typing import List from typing import Tuple -from configuration_kind import ConfigurationKind from fail_kind import FailKind -from params import Params, TestReport +from params import TestReport from test_base import Test @@ -27,23 +24,18 @@ class TestETS(Test): if test_basename.startswith("n."): is_negative_test = True - report, compiled_file, fail_kind = self._run_compiler() - self.fail_kind= fail_kind - self.report = report - if report.return_code != 0: + self.report, compiled_file, self.fail_kind = self._run_compiler() + if self.report.return_code != 0: self.passed = False - return self self.report, self.fail_kind = self._run_runtime(compiled_file) if self.report.return_code != 0 and is_negative_test: self.passed = True - return self elif self.report.return_code != 0: self.passed = False - return self else: self.passed = True - return self + return self def _run_compiler(self) -> Tuple[TestReport,str,FailKind]: -- Gitee From c0eb01e0119c13cf3850efde1951b2e2aaeb85fb Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 17:51:00 +0300 Subject: [PATCH 31/36] fix Test Env object initialzation --- test/runner/params.py | 18 +++++++++--------- test/runner/runner_ets.py | 6 +++--- test/runner/test_ets.py | 1 + 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/test/runner/params.py b/test/runner/params.py index ce108b22e..2b14dc473 100644 --- a/test/runner/params.py +++ b/test/runner/params.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, List from configuration_kind import ConfigurationKind @@ -14,17 +14,17 @@ class TestEnv: cmd_prefix: List[str] cmd_env: Any - es2panda: str - es2panda_args : List[str] + es2panda: str = "" + es2panda_args : List[str] = field(default_factory=list) - runtime: str - runtime_args: List[str] + runtime: str = "" + runtime_args: List[str] = field(default_factory=list) - arkaout: str - aot_args: List[str] + arkaout: str = "" + aot_args: List[str] = field(default_factory=list) - ark_quick: str - quick_args: List[str] + ark_quick: str = "" + quick_args: List[str] = field(default_factory=list) util: Any = None diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 24cbb746c..dc492be41 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -31,11 +31,11 @@ class RunnerETS(RunnerFileBased): "--opt-level=0"], runtime=self.runtime, runtime_args=[f"--boot-panda-files={self.stdlib_path}", "--load-runtimes=ets"], - arkaout=[], + arkaout="", aot_args=[], - ark_quick=[], + ark_quick="", quick_args=[], - conf_kind=[], + conf_kind= None, cmd_prefix=[] ) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 61ad11381..8dcbed244 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -27,6 +27,7 @@ class TestETS(Test): self.report, compiled_file, self.fail_kind = self._run_compiler() if self.report.return_code != 0: self.passed = False + return self self.report, self.fail_kind = self._run_runtime(compiled_file) if self.report.return_code != 0 and is_negative_test: -- Gitee From 26a0f522d1bc65306d0c8d24feb648c98cc520db Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 18:02:44 +0300 Subject: [PATCH 32/36] TestEnv default initialization fixed --- test/runner/params.py | 8 +++----- test/runner/runner_ets.py | 8 +------- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/test/runner/params.py b/test/runner/params.py index 2b14dc473..5e5a16748 100644 --- a/test/runner/params.py +++ b/test/runner/params.py @@ -8,12 +8,10 @@ from fail_kind import FailKind @dataclass class TestEnv: args: Any - - conf_kind: ConfigurationKind - - cmd_prefix: List[str] cmd_env: Any - + conf_kind: ConfigurationKind + cmd_prefix: List[str] = field(default_factory=list) + es2panda: str = "" es2panda_args : List[str] = field(default_factory=list) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index dc492be41..cfa548486 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -4,7 +4,6 @@ from pathlib import Path from params import TestEnv from runner_file_based import RunnerFileBased -from utils import write_2_file class RunnerETS(RunnerFileBased): @@ -31,12 +30,7 @@ class RunnerETS(RunnerFileBased): "--opt-level=0"], runtime=self.runtime, runtime_args=[f"--boot-panda-files={self.stdlib_path}", "--load-runtimes=ets"], - arkaout="", - aot_args=[], - ark_quick="", - quick_args=[], - conf_kind= None, - cmd_prefix=[] + conf_kind= self.conf_kind, ) def _check_binary_artefacts(self) : -- Gitee From 5ee91799959af46c536034196dce1b046cf24957 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Wed, 26 Oct 2022 14:27:46 +0300 Subject: [PATCH 33/36] add support es2panda custom folder --- test/runner/runner_file_based.py | 6 +++++- test/runner/starter.py | 9 +++++++-- test/runner/test_ets.py | 6 +++--- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/test/runner/runner_file_based.py b/test/runner/runner_file_based.py index 715219d48..d2c61ffce 100644 --- a/test/runner/runner_file_based.py +++ b/test/runner/runner_file_based.py @@ -50,7 +50,11 @@ class RunnerFileBased(Runner): # we don't want to interpret asan failures as SyntaxErrors self.cmd_env[san] = ":exitcode=255" - self.es2panda = path.join(self.build_dir, "bin", "es2panda") + if args.custom_es2panda_path: + self.es2panda = args.custom_es2panda_path + else: + self.es2panda = path.join(self.build_dir, "bin", "es2panda") + if not path.isfile(self.es2panda): raise Exception(f"Cannot find es2panda binary: {self.es2panda}") diff --git a/test/runner/starter.py b/test/runner/starter.py index 62e2eb17e..736db9aaa 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -44,8 +44,13 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', - default=False, help='run test against ETS specification') + parser.add_argument( + '--ets-stdlib', action='store_true', dest='ets_stdlib', + default=False, help='run test against ETS STDLIB') + parser.add_argument( + '--custom-es2panda', action='store', dest='custom_es2panda_path', + type=lambda arg: is_file(parser, arg), + default=False, help='custom path to es2panda binary file') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), help='directory with test file. If not set the module directory is used') diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 8dcbed244..87e3f65e4 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -4,12 +4,12 @@ from typing import Tuple from fail_kind import FailKind from params import TestReport -from test_base import Test +from test_js import TestJS -class TestETS(Test): +class TestETS(TestJS): def __init__(self, test_env, test_path, flags, test_id, update_expected=False): - Test.__init__(self, test_env, test_path, + TestJS.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further -- Gitee From c57f2d25bca63a588a9f5e694aff5e0682ff8c54 Mon Sep 17 00:00:00 2001 From: Shirunova Viktoria Date: Wed, 26 Oct 2022 13:21:39 +0300 Subject: [PATCH 34/36] proposed changes to reuse TestJS --- test/runner/runner_ets.py | 41 +++-------- test/runner/runner_ets_stdlib.py | 15 ++-- test/runner/test_ets.py | 116 ++++++++++++++----------------- 3 files changed, 70 insertions(+), 102 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index cfa548486..0934ae127 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -1,8 +1,6 @@ -from os import path, environ +from os import path from pathlib import Path - -from params import TestEnv from runner_file_based import RunnerFileBased @@ -11,42 +9,23 @@ class RunnerETS(RunnerFileBased): RunnerFileBased.__init__(self, args, name) self.stdlib_path = path.join(self.test_root, "stdlib") self.stdlib_name = "etsstdlib.abc" - self.cmd_env = environ.copy() - self.test_root = args.test_root - self.compiler = path.join(self.build_dir, "bin", "es2panda") - self.runtime = path.join(self.build_dir, "bin", "ark") - self.cmd_env = environ.copy() self.stdlib_path = path.join(self.build_dir, "etsstdlib.abc") self.stdlib_src_path = path.join(self.test_root, "stdlib") self._check_binary_artefacts() - - self.test_env = TestEnv( - args=args, - cmd_env=self.cmd_env, - es2panda =self.compiler, - es2panda_args= [f"--stdlib={self.stdlib_src_path}", - "--gen-stdlib=false", - "--extension=ets", - "--opt-level=0"], - runtime=self.runtime, - runtime_args=[f"--boot-panda-files={self.stdlib_path}", "--load-runtimes=ets"], - conf_kind= self.conf_kind, - ) - def _check_binary_artefacts(self) : - compiler_path_obj = Path(self.compiler) - runtime_path_obj = Path(self.runtime) + self.test_env.es2panda_args = [f"--stdlib={self.stdlib_src_path}", + "--gen-stdlib=false", + "--extension=ets", + "--opt-level=0"] + self.test_env.runtime_args = [f"--boot-panda-files={self.stdlib_path}", + "--load-runtimes=ets"] + + def _check_binary_artefacts(self): stdlib_path_obj = Path(self.stdlib_path) stdlib_src_path_obj = Path(self.stdlib_src_path) - if not compiler_path_obj.is_file(): - raise FileNotFoundError("Hmm ... ETS Compiler was not found") - - if not runtime_path_obj.is_file(): - raise FileNotFoundError("Hmm ... ETS Runtime was not found") - if not stdlib_path_obj.is_file(): raise FileNotFoundError("Hmm ... standard library abc file not found") if not stdlib_src_path_obj.is_dir(): - raise FileNotFoundError("Hmm ... Source code of standart library was not found") \ No newline at end of file + raise FileNotFoundError("Hmm ... Source code of standard library was not found") diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index 1f0fcab74..07cd01119 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,24 +1,21 @@ import os + +from runner_base import get_test_id from runner_ets import RunnerETS from test_ets import TestETS -from runner_base import correct_path, get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-stdlib") self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") - self.ignored_name_prefix = self.name - self.excluded_lists = [correct_path(self.list_root, f"{self.ignored_name_prefix}-excluded.txt")] + + self.collect_excluded_test_lists() + self.collect_ignored_test_lists() + self.add_directory(self.stdlib_test_path, "ets", []) - def create_test(self, test_file, flags, is_ignored): test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) test.ignored = is_ignored return test - - - - - \ No newline at end of file diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 87e3f65e4..deb309ccd 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -1,94 +1,86 @@ -import subprocess from os import path, makedirs from typing import Tuple from fail_kind import FailKind -from params import TestReport +from params import Params, TestReport from test_js import TestJS class TestETS(TestJS): def __init__(self, test_env, test_path, flags, test_id, update_expected=False): - TestJS.__init__(self, test_env, test_path, - flags, test_id, update_expected) + TestJS.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None self.ets_main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) - - def do_run(self): test_basename = path.basename(self.path) - is_negative_test = False - if test_basename.startswith("n."): - is_negative_test = True + self.is_negative_test = test_basename.startswith("n.") - self.report, compiled_file, self.fail_kind = self._run_compiler() - if self.report.return_code != 0: - self.passed = False + def do_run(self): + self.passed, self.report, compiled_file, self.fail_kind = self._run_compiler() + if not self.passed: return self - self.report, self.fail_kind = self._run_runtime(compiled_file) - if self.report.return_code != 0 and is_negative_test: - self.passed = True - elif self.report.return_code != 0: - self.passed = False - else: - self.passed = True + self.passed, self.report, self.fail_kind = self._run_runtime(compiled_file) return self + def _runtime_result_validator(self, return_code: int) -> bool: + """ + :return: True if test is successful, False if failed + """ + if self.is_negative_test: + return return_code != 0 + else: + return return_code == 0 - def _run_compiler(self) -> Tuple[TestReport,str,FailKind]: - fail_kind = None + def _run_compiler(self) -> Tuple[bool, TestReport, str, FailKind]: test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - compiler_cmd_string = [] - compiler_cmd_string.append(self.test_env.es2panda) - compiler_cmd_string.extend(self.test_env.es2panda_args) - compiler_cmd_string.append(f"--output={output_path}") - compiler_cmd_string.append(self.path) - self.log_cmd(compiler_cmd_string) - process = subprocess.Popen(compiler_cmd_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - stdout_content, stderr_content = process.communicate(timeout=60) - except subprocess.TimeoutExpired: - process.kill() - fail_kind = FailKind.ES2PANDA_TIMEOUT - if process.returncode != 0: - fail_kind = FailKind.ES2PANDA_FAIL + es2panda_flags = [] + es2panda_flags.extend(self.test_env.es2panda_args) + es2panda_flags.append(f"--output={output_path}") + es2panda_flags.append(self.path) - report = TestReport( - output=stdout_content.decode('UTF-8'), - error=stderr_content.decode('UTF-8'), - return_code=process.returncode + params = Params( + executor=self.test_env.es2panda, + flags=es2panda_flags, + env=self.test_env.cmd_env, + timeout=self.test_env.args.es2panda_timeout, + fail_kind_fail=FailKind.ES2PANDA_FAIL, + fail_kind_timeout=FailKind.ES2PANDA_TIMEOUT, + fail_kind_other=FailKind.ES2PANDA_OTHER, ) - - return (report, output_path, fail_kind) - def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: - fail_kind = None - runtime_cmd_string = [] - runtime_cmd_string.append(self.test_env.runtime) - runtime_cmd_string.extend(self.test_env.runtime_args) - runtime_cmd_string.append(abc_file) - runtime_cmd_string.append(self.ets_main_entry_point) - self.log_cmd(runtime_cmd_string) - process = subprocess.Popen(runtime_cmd_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - stdout_content, stderr_content = process.communicate(timeout=60) - except subprocess.TimeoutExpired: - process.kill() - fail_kind = FailKind.RUNTIME_TIMEOUT + passed, report, output_path, fail_kind = self.run_one_step( + name="es2panda", + params=params, + result_validator=lambda _, _2, rc: rc == 0 + ) + + return (passed, report, output_path, fail_kind) - if process.returncode != 0: - fail_kind = FailKind.RUNTIME_FAIL + def _run_runtime(self, abc_file: str) -> Tuple[bool, TestReport, FailKind]: + ark_flags = [] + ark_flags.extend(self.test_env.runtime_args) + ark_flags.append(abc_file) + ark_flags.append(self.ets_main_entry_point) + + params = Params( + timeout=self.test_env.args.timeout, + executor=self.test_env.runtime, + flags=ark_flags, + env=self.test_env.cmd_env, + fail_kind_fail=FailKind.RUNTIME_FAIL, + fail_kind_timeout=FailKind.RUNTIME_TIMEOUT, + fail_kind_other=FailKind.RUNTIME_OTHER, + ) - runtime_report = TestReport( - output=stdout_content.decode('UTF-8'), - error=stderr_content.decode('UTF-8'), - return_code=process.returncode + return self.run_one_step( + name="ark", + params=params, + result_validator=lambda _, _2, rc: self._runtime_result_validator(rc) ) - return runtime_report, fail_kind \ No newline at end of file -- Gitee From ae03f9bab2db16c28d96665b37b9c495261e3e39 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Wed, 26 Oct 2022 15:04:33 +0300 Subject: [PATCH 35/36] review has been completed --- test/runner/test_ets.py | 41 ++++++++++++++++------------------------- test/runner/test_js.py | 3 ++- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index deb309ccd..474c8fef7 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -3,6 +3,7 @@ from typing import Tuple from fail_kind import FailKind from params import Params, TestReport +from configuration_kind import ConfigurationKind from test_js import TestJS @@ -12,18 +13,30 @@ class TestETS(TestJS): # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None - self.ets_main_entry_point = "ETSGLOBAL::main" + self.main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) test_basename = path.basename(self.path) self.is_negative_test = test_basename.startswith("n.") + self.test_an = path.join(self.bytecode_path, f"{self.test_id}.an") def do_run(self): self.passed, self.report, compiled_file, self.fail_kind = self._run_compiler() if not self.passed: return self - self.passed, self.report, self.fail_kind = self._run_runtime(compiled_file) + # Run aot if required + if self.test_env.conf_kind in [ConfigurationKind.AOT, ConfigurationKind.AOT_FULL]: + self.passed, self.report, self.fail_kind = self.run_aot( + self.test_an, + compiled_file, + lambda o, e, rc: rc == 0 + ) + + if not self.passed: + return self + + self.passed, self.report, self.fail_kind = self.run_runtime(self.test_an, compiled_file, lambda _, _2, rc : self._runtime_result_validator(rc) ) return self def _runtime_result_validator(self, return_code: int) -> bool: @@ -55,32 +68,10 @@ class TestETS(TestJS): fail_kind_other=FailKind.ES2PANDA_OTHER, ) - passed, report, output_path, fail_kind = self.run_one_step( + passed, report, fail_kind = self.run_one_step( name="es2panda", params=params, result_validator=lambda _, _2, rc: rc == 0 ) return (passed, report, output_path, fail_kind) - - def _run_runtime(self, abc_file: str) -> Tuple[bool, TestReport, FailKind]: - ark_flags = [] - ark_flags.extend(self.test_env.runtime_args) - ark_flags.append(abc_file) - ark_flags.append(self.ets_main_entry_point) - - params = Params( - timeout=self.test_env.args.timeout, - executor=self.test_env.runtime, - flags=ark_flags, - env=self.test_env.cmd_env, - fail_kind_fail=FailKind.RUNTIME_FAIL, - fail_kind_timeout=FailKind.RUNTIME_TIMEOUT, - fail_kind_other=FailKind.RUNTIME_OTHER, - ) - - return self.run_one_step( - name="ark", - params=params, - result_validator=lambda _, _2, rc: self._runtime_result_validator(rc) - ) diff --git a/test/runner/test_js.py b/test/runner/test_js.py index 589ec13ca..fb05a128b 100644 --- a/test/runner/test_js.py +++ b/test/runner/test_js.py @@ -15,6 +15,7 @@ class TestJS(Test): # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None + self.main_entry_point = "_GLOBAL::func_main_0" def run_one_step(self, name, params: Params, result_validator): cmd = self.test_env.cmd_prefix + [params.executor] @@ -91,7 +92,7 @@ class TestJS(Test): if self.test_env.conf_kind == ConfigurationKind.IRTOC: ark_flags.extend(['--interpreter-type=irtoc']) - ark_flags.extend([test_abc, "_GLOBAL::func_main_0"]) + ark_flags.extend([test_abc, self.main_entry_point]) params = Params( timeout=self.test_env.args.timeout, -- Gitee From 8cc079a86799c26d5ae17432948aeaabebc42faa Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 16:26:26 +0300 Subject: [PATCH 36/36] replace exception with file not found exception --- test/runner/params.py | 25 +++---- test/runner/runner_ets.py | 60 +++------------ test/runner/runner_ets_stdlib.py | 17 ++--- test/runner/runner_file_based.py | 7 +- test/runner/starter.py | 9 ++- test/runner/test_ets.py | 124 +++++++++++++------------------ test/runner/test_js.py | 3 +- 7 files changed, 93 insertions(+), 152 deletions(-) diff --git a/test/runner/params.py b/test/runner/params.py index df994ed45..5e5a16748 100644 --- a/test/runner/params.py +++ b/test/runner/params.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, List from configuration_kind import ConfigurationKind @@ -8,22 +8,21 @@ from fail_kind import FailKind @dataclass class TestEnv: args: Any - - conf_kind: ConfigurationKind - - cmd_prefix: List[str] cmd_env: Any + conf_kind: ConfigurationKind + cmd_prefix: List[str] = field(default_factory=list) + + es2panda: str = "" + es2panda_args : List[str] = field(default_factory=list) - es2panda: str - - runtime: str - runtime_args: List[str] + runtime: str = "" + runtime_args: List[str] = field(default_factory=list) - arkaout: str - aot_args: List[str] + arkaout: str = "" + aot_args: List[str] = field(default_factory=list) - ark_quick: str - quick_args: List[str] + ark_quick: str = "" + quick_args: List[str] = field(default_factory=list) util: Any = None diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index f89444187..0934ae127 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -1,15 +1,7 @@ -from cProfile import run -from os import path, environ +from os import path from pathlib import Path - -from configuration_kind import ConfigurationKind -from fail_kind import FailKind -from params import TestEnv -from report_format import ReportFormat -from runner_base import Runner from runner_file_based import RunnerFileBased -from utils import write_2_file class RunnerETS(RunnerFileBased): @@ -17,55 +9,23 @@ class RunnerETS(RunnerFileBased): RunnerFileBased.__init__(self, args, name) self.stdlib_path = path.join(self.test_root, "stdlib") self.stdlib_name = "etsstdlib.abc" - stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) - self.cmd_env = environ.copy() - self.test_root = args.test_root - self.compiler = path.join(self.build_dir, "bin", "es2panda") - self.runtime = path.join(self.build_dir, "bin", "ark") - self.cmd_env = environ.copy() self.stdlib_path = path.join(self.build_dir, "etsstdlib.abc") self.stdlib_src_path = path.join(self.test_root, "stdlib") self._check_binary_artefacts() - - - stdlib_file = Path(stdlib_output_path) - # if not stdlib_file.exists(): - # self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) - self.test_env = TestEnv( - args=args, - cmd_env=self.cmd_env, - es2panda =self.compiler, - runtime=self.runtime, - runtime_args=[f"--boot-panda-files={self.stdlib_path}", "--load-runtimes=ets"], - arkaout="", - aot_args="", - ark_quick="", - quick_args="", - conf_kind="", - cmd_prefix="" - ) - - self.test_env.es2panda_args= [f"--stdlib={self.stdlib_src_path}", - "--gen-stdlib=false", - "--extension=ets", - "--opt-level=0"] + self.test_env.es2panda_args = [f"--stdlib={self.stdlib_src_path}", + "--gen-stdlib=false", + "--extension=ets", + "--opt-level=0"] + self.test_env.runtime_args = [f"--boot-panda-files={self.stdlib_path}", + "--load-runtimes=ets"] - - def _check_binary_artefacts(self) : - compiler_path_obj = Path(self.compiler) - runtime_path_obj = Path(self.runtime) + def _check_binary_artefacts(self): stdlib_path_obj = Path(self.stdlib_path) stdlib_src_path_obj = Path(self.stdlib_src_path) - if not compiler_path_obj.is_file(): - raise Exception("Hmm ... ETS Compiler was not found") - - if not runtime_path_obj.is_file(): - raise Exception("Hmm ... ETS Runtime was not found") - if not stdlib_path_obj.is_file(): - raise Exception("Hmm ... standart library abc file not found") + raise FileNotFoundError("Hmm ... standard library abc file not found") if not stdlib_src_path_obj.is_dir(): - raise Exception("Hmm ... Source code of standart library was not found") \ No newline at end of file + raise FileNotFoundError("Hmm ... Source code of standard library was not found") diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index 4c384d6e1..07cd01119 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,26 +1,21 @@ -import shlex import os -import subprocess + +from runner_base import get_test_id from runner_ets import RunnerETS from test_ets import TestETS -from runner_base import correct_path, get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-stdlib") self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") - self.ignored_name_prefix = self.name - self.excluded_lists = [correct_path(self.list_root, f"{self.ignored_name_prefix}-excluded.txt")] + + self.collect_excluded_test_lists() + self.collect_ignored_test_lists() + self.add_directory(self.stdlib_test_path, "ets", []) - def create_test(self, test_file, flags, is_ignored): test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) test.ignored = is_ignored return test - - - - - \ No newline at end of file diff --git a/test/runner/runner_file_based.py b/test/runner/runner_file_based.py index 410d6faf9..d2c61ffce 100644 --- a/test/runner/runner_file_based.py +++ b/test/runner/runner_file_based.py @@ -50,7 +50,11 @@ class RunnerFileBased(Runner): # we don't want to interpret asan failures as SyntaxErrors self.cmd_env[san] = ":exitcode=255" - self.es2panda = path.join(self.build_dir, "bin", "es2panda") + if args.custom_es2panda_path: + self.es2panda = args.custom_es2panda_path + else: + self.es2panda = path.join(self.build_dir, "bin", "es2panda") + if not path.isfile(self.es2panda): raise Exception(f"Cannot find es2panda binary: {self.es2panda}") @@ -113,6 +117,7 @@ class RunnerFileBased(Runner): cmd_prefix=self.cmd_prefix, cmd_env=self.cmd_env, es2panda=self.es2panda, + es2panda_args=[], runtime=self.runtime, runtime_args=self.runtime_args, arkaout=self.arkaot, diff --git a/test/runner/starter.py b/test/runner/starter.py index 5853fc662..736db9aaa 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -44,8 +44,13 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', - default=False, help='run test agaisnt ETS specification') + parser.add_argument( + '--ets-stdlib', action='store_true', dest='ets_stdlib', + default=False, help='run test against ETS STDLIB') + parser.add_argument( + '--custom-es2panda', action='store', dest='custom_es2panda_path', + type=lambda arg: is_file(parser, arg), + default=False, help='custom path to es2panda binary file') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), help='directory with test file. If not set the module directory is used') diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 66d137b4b..474c8fef7 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -1,101 +1,77 @@ -import subprocess -import sys from os import path, makedirs -from typing import List from typing import Tuple -from configuration_kind import ConfigurationKind from fail_kind import FailKind from params import Params, TestReport -from test_base import Test +from configuration_kind import ConfigurationKind +from test_js import TestJS -class TestETS(Test): +class TestETS(TestJS): def __init__(self, test_env, test_path, flags, test_id, update_expected=False): - Test.__init__(self, test_env, test_path, - flags, test_id, update_expected) + TestJS.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None - self.ets_main_entry_point = "ETSGLOBAL::main" + self.main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) - - def do_run(self): test_basename = path.basename(self.path) - is_negative_test = False - if test_basename.startswith("n."): - is_negative_test = True + self.is_negative_test = test_basename.startswith("n.") + self.test_an = path.join(self.bytecode_path, f"{self.test_id}.an") - report, compiled_file, fail_kind = self._run_compiler() - self.fail_kind= fail_kind - self.report = report - if report.return_code != 0: - self.passed = False + def do_run(self): + self.passed, self.report, compiled_file, self.fail_kind = self._run_compiler() + if not self.passed: return self - self.report, self.fail_kind = self._run_runtime(compiled_file) - if self.report.return_code != 0 and is_negative_test: - self.passed = True - return self - elif self.report.return_code != 0: - self.passed = False - return self - else: - self.passed = True - return self + # Run aot if required + if self.test_env.conf_kind in [ConfigurationKind.AOT, ConfigurationKind.AOT_FULL]: + self.passed, self.report, self.fail_kind = self.run_aot( + self.test_an, + compiled_file, + lambda o, e, rc: rc == 0 + ) + if not self.passed: + return self + + self.passed, self.report, self.fail_kind = self.run_runtime(self.test_an, compiled_file, lambda _, _2, rc : self._runtime_result_validator(rc) ) + return self + + def _runtime_result_validator(self, return_code: int) -> bool: + """ + :return: True if test is successful, False if failed + """ + if self.is_negative_test: + return return_code != 0 + else: + return return_code == 0 - def _run_compiler(self) -> Tuple[TestReport,str,FailKind]: - fail_kind = None + def _run_compiler(self) -> Tuple[bool, TestReport, str, FailKind]: test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - compiler_cmd_string = [] - compiler_cmd_string.append(self.test_env.es2panda) - compiler_cmd_string.extend(self.test_env.es2panda_args) - compiler_cmd_string.append(f"--output={output_path}") - compiler_cmd_string.append(self.path) - self.log_cmd(compiler_cmd_string) - process = subprocess.Popen(compiler_cmd_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - stdout_content, stderr_content = process.communicate(timeout=60) - except subprocess.TimeoutExpired: - process.kill() - fail_kind = FailKind.ES2PANDA_TIMEOUT - if process.returncode != 0: - fail_kind = FailKind.ES2PANDA_FAIL + es2panda_flags = [] + es2panda_flags.extend(self.test_env.es2panda_args) + es2panda_flags.append(f"--output={output_path}") + es2panda_flags.append(self.path) - report = TestReport( - output=stdout_content.decode('UTF-8'), - error=stderr_content.decode('UTF-8'), - return_code=process.returncode + params = Params( + executor=self.test_env.es2panda, + flags=es2panda_flags, + env=self.test_env.cmd_env, + timeout=self.test_env.args.es2panda_timeout, + fail_kind_fail=FailKind.ES2PANDA_FAIL, + fail_kind_timeout=FailKind.ES2PANDA_TIMEOUT, + fail_kind_other=FailKind.ES2PANDA_OTHER, ) - - return (report, output_path, fail_kind) - def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: - fail_kind = None - runtime_cmd_string = [] - runtime_cmd_string.append(self.test_env.runtime) - runtime_cmd_string.extend(self.test_env.runtime_args) - runtime_cmd_string.append(abc_file) - runtime_cmd_string.append(self.ets_main_entry_point) - self.log_cmd(runtime_cmd_string) - process = subprocess.Popen(runtime_cmd_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - stdout_content, stderr_content = process.communicate(timeout=60) - except subprocess.TimeoutExpired: - process.kill() - fail_kind = FailKind.RUNTIME_TIMEOUT - - if process.returncode != 0: - fail_kind = FailKind.RUNTIME_FAIL - - runtime_report = TestReport( - output=stdout_content.decode('UTF-8'), - error=stderr_content.decode('UTF-8'), - return_code=process.returncode + passed, report, fail_kind = self.run_one_step( + name="es2panda", + params=params, + result_validator=lambda _, _2, rc: rc == 0 ) - return runtime_report, fail_kind \ No newline at end of file + + return (passed, report, output_path, fail_kind) diff --git a/test/runner/test_js.py b/test/runner/test_js.py index 589ec13ca..fb05a128b 100644 --- a/test/runner/test_js.py +++ b/test/runner/test_js.py @@ -15,6 +15,7 @@ class TestJS(Test): # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None + self.main_entry_point = "_GLOBAL::func_main_0" def run_one_step(self, name, params: Params, result_validator): cmd = self.test_env.cmd_prefix + [params.executor] @@ -91,7 +92,7 @@ class TestJS(Test): if self.test_env.conf_kind == ConfigurationKind.IRTOC: ark_flags.extend(['--interpreter-type=irtoc']) - ark_flags.extend([test_abc, "_GLOBAL::func_main_0"]) + ark_flags.extend([test_abc, self.main_entry_point]) params = Params( timeout=self.test_env.args.timeout, -- Gitee