From 9d3bb64b591e2cf42d06d985bc715cef2d2851d3 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 14:29:45 +0300 Subject: [PATCH 01/12] The first ETS test running structure --- test/runner/runner_ets.py | 37 +++++++++++++++++++++++++++++++++++ test/runner/runner_ets_cts.py | 12 ++++++++++++ test/runner/test_ets.py | 17 ++++++++++++++++ test/runner/test_ets_cts.py | 13 ++++++++++++ 4 files changed, 79 insertions(+) create mode 100644 test/runner/runner_ets.py create mode 100644 test/runner/runner_ets_cts.py create mode 100644 test/runner/test_ets.py create mode 100644 test/runner/test_ets_cts.py diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py new file mode 100644 index 000000000..de3b5bc1c --- /dev/null +++ b/test/runner/runner_ets.py @@ -0,0 +1,37 @@ +import subprocess +import sys +from datetime import datetime +from glob import glob +from os import path, environ, makedirs +from typing import List + +from configuration_kind import ConfigurationKind +from fail_kind import FailKind +from params import TestEnv +from report_format import ReportFormat +from runner_base import Runner +from utils import write_2_file + +INDEX_TITLE = "${Title}" +INDEX_OPTIONS = "${Options}" +INDEX_TOTAL = "${Total}" +INDEX_PASSED = "${Passed}" +INDEX_FAILED = "${Failed}" +INDEX_IGNORED = "${Ignored}" +INDEX_EXCLUDED_LISTS = "${ExcludedThroughLists}" +INDEX_EXCLUDED_OTHER = "${ExcludedByOtherReasons}" +INDEX_TEST_NAME = "${TestName}" +INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" + + +class RunnerETS(Runner): + def __init__(self, args, name): + Runner.__init__(self, args, name) + self.cmd_env = environ.copy() + + def _generate_ets_stdlib(): + pass + + + + \ No newline at end of file diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py new file mode 100644 index 000000000..daa27ad29 --- /dev/null +++ b/test/runner/runner_ets_cts.py @@ -0,0 +1,12 @@ +from runner_base import correct_path, get_test_id +from runner_ets import RunnerETS +from test_ets_cts import TestJSHermes +from util_hermes import UtilHermes + + +class RunnerETSCTS(RunnerJS): + def __init__(self, args): + RunnerETS.__init__(self, args, "hermes") + + + \ No newline at end of file diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py new file mode 100644 index 000000000..8ec62c3d5 --- /dev/null +++ b/test/runner/test_ets.py @@ -0,0 +1,17 @@ +import subprocess +import sys +from os import path, remove +from typing import List + +from configuration_kind import ConfigurationKind +from fail_kind import FailKind +from params import Params, TestReport +from test_base import Test + + +class TestETS(Test): + def __init__(self, test_env, test_path, flags, test_id, update_expected): + Test.__init__(self, test_env, test_path, flags, test_id, update_expected) + # If test fails it contains reason (of FailKind enum) of first failed step + # It's supposed if the first step is failed then no step is executed further + self.fail_kind = None \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py new file mode 100644 index 000000000..80e421fa8 --- /dev/null +++ b/test/runner/test_ets_cts.py @@ -0,0 +1,13 @@ +from os import path, makedirs + +from configuration_kind import ConfigurationKind +from test_ets import TestETS +from utils import purify + +class TestETS_CTS(TestETS): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) + self.tmp_dir = path.join(path.sep, "tmp", "cts") + makedirs(self.tmp_dir, exist_ok=True) + self.util = self.test_env.util + -- Gitee From eff1d656c0ad345455016f55266b4551b86beead Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 14:29:51 +0300 Subject: [PATCH 02/12] add etc cts specific runner --- test/runner/runner_ets_cts.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index daa27ad29..db57261d1 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -4,9 +4,13 @@ from test_ets_cts import TestJSHermes from util_hermes import UtilHermes -class RunnerETSCTS(RunnerJS): +class RunnerETSCTS(RunnerETS): def __init__(self, args): - RunnerETS.__init__(self, args, "hermes") + RunnerETS.__init__(self, args, "ets-cts") + + def _generate_ets_code_from_template(): + pass + \ No newline at end of file -- Gitee From a28a5fecaf460c0b6daffb779cbcc3c3db7fffe8 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Mon, 17 Oct 2022 18:32:28 +0300 Subject: [PATCH 03/12] add stdlib structure for runner --- test/runner/runner.py | 8 ++++++++ test/runner/runner_ets_cts.py | 5 ++--- test/runner/runner_ets_stdlib.py | 15 +++++++++++++++ test/runner/starter.py | 8 ++++++-- test/runner/test_ets_stdlib.py | 13 +++++++++++++ 5 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 test/runner/runner_ets_stdlib.py create mode 100644 test/runner/test_ets_stdlib.py diff --git a/test/runner/runner.py b/test/runner/runner.py index 607e8cb25..74ca8960a 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -4,6 +4,8 @@ from dotenv import load_dotenv from runner_js_hermes import RunnerJSHermes from runner_js_parser import RunnerJSParser from runner_js_test262 import RunnerJSTest262 +from runner_ets_cts import RunnerETS_CTS +from runner_ets_stdlib import RunnerETS_STDLIB from starter import get_args @@ -27,6 +29,12 @@ def main(): if args.hermes: runners.append(RunnerJSHermes(args)) + if args.ets_cts: + runners.append(RunnerETS_CTS(args)) + + if args.ets_stdlib: + runners.append(RunnerETS_STDLIB(args)) + failed_tests = 0 for runner in runners: diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index db57261d1..55b0dafbe 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -1,10 +1,9 @@ from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_cts import TestJSHermes -from util_hermes import UtilHermes +from test_ets_cts import TestETS_CTS -class RunnerETSCTS(RunnerETS): +class RunnerETS_CTS(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-cts") diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py new file mode 100644 index 000000000..cf801d317 --- /dev/null +++ b/test/runner/runner_ets_stdlib.py @@ -0,0 +1,15 @@ +from runner_base import correct_path, get_test_id +from runner_ets import RunnerETS +from test_ets_stdlib import TestETS_STDLIB + + +class RunnerETS_STDLIB(RunnerETS): + def __init__(self, args): + RunnerETS.__init__(self, args, "ets-cts") + + def _generate_ets_code_from_template(): + pass + + + + \ No newline at end of file diff --git a/test/runner/starter.py b/test/runner/starter.py index a1f1792f1..553faa5f3 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -21,7 +21,8 @@ def is_file(parser, arg): def check_timeout(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError(f"{value} is an invalid timeout value") + raise argparse.ArgumentTypeError( + f"{value} is an invalid timeout value") return ivalue @@ -43,7 +44,10 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - + parser.add_argument('--ets-cts', action='store_true', dest='ets_sts', + default=False, help='run test agaisnt ETS specification') + parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', + default=False, help='run test agaisnt ETS specification') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), help='directory with test file. If not set the module directory is used') diff --git a/test/runner/test_ets_stdlib.py b/test/runner/test_ets_stdlib.py new file mode 100644 index 000000000..dd1b3fb07 --- /dev/null +++ b/test/runner/test_ets_stdlib.py @@ -0,0 +1,13 @@ +from os import path, makedirs + +from configuration_kind import ConfigurationKind +from test_ets import TestETS +from utils import purify + +class TestETS_STDLIB(TestETS): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) + self.tmp_dir = path.join(path.sep, "tmp", "cts") + makedirs(self.tmp_dir, exist_ok=True) + self.util = self.test_env.util + -- Gitee From e6f4068e23827302b0c233c26696f783987ad303 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 18 Oct 2022 18:16:59 +0300 Subject: [PATCH 04/12] add stdlib generating, cts code generating --- test/runner/runner_ets.py | 47 +++++++++++++++++++++++++++++++---- test/runner/runner_ets_cts.py | 32 ++++++++++++++++++++++-- test/runner/starter.py | 2 +- test/runner/test_ets.py | 12 +++++++-- test/runner/test_ets_cts.py | 4 +-- 5 files changed, 85 insertions(+), 12 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index de3b5bc1c..0f028d6e3 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -4,6 +4,9 @@ from datetime import datetime from glob import glob from os import path, environ, makedirs from typing import List +from pathlib import Path +import shlex + from configuration_kind import ConfigurationKind from fail_kind import FailKind @@ -25,13 +28,47 @@ INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" class RunnerETS(Runner): - def __init__(self, args, name): + def __init__(self, args, name:str): Runner.__init__(self, args, name) + output_path = path.join(self.stdlib_path, self.stdlib_name) + self.cmd_env = environ.copy() + self.test_root=args.test_root + self.compiler=path.join(self.build_dir, "bin", "es2panda") + self.runtime = path.join(self.build_dir, "bin", "ark") + self.runtime_args = f"--boot-panda={output_path} --load_runtime=ets" self.cmd_env = environ.copy() + self.stdlib_name="etsstdlib.abc" + self.bytecode_path=path.join("/tmp", "ets", "cts") + self.stdlib_path=path.join(self.test_root, "stdlib") + self.compiler_args = f"--stdlib={self.stdlib_path} --gen-stdlib=false --extension=ets --opt_level=0" + + stdlib_file = Path(output_path) + if not stdlib_file.exists(): + self._generate_ets_stdlib(self.stdlib_path, output_path) + self.test_env = TestEnv( + args=args, + cmd_env=self.cmd_env, + compiler=self.compiler, + runtime=self.runtime, + compiler_args = self.compiler_args, + runtime_args=self.runtime_args, + stdlib_path=self.stdlib_path, + bytecode_path=self.bytecode_path + ) + + def _generate_ets_stdlib(self, stdlib_path : str, output_path : str): + dummy_test=path.join(self.test_root, "stdlib/std/core/Boolean.ets") + options=f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" + cmd_string=f"{self.compiler} {options}" + args=shlex.split(cmd_string) + process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) - def _generate_ets_stdlib(): - pass + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Cannot generate ETS stdlib : timeout") - + if process.returncode != 0: + raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - \ No newline at end of file diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index 55b0dafbe..aba767ab7 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -1,3 +1,6 @@ +import os.path +import shlex +import subprocess from runner_base import correct_path, get_test_id from runner_ets import RunnerETS from test_ets_cts import TestETS_CTS @@ -6,9 +9,34 @@ from test_ets_cts import TestETS_CTS class RunnerETS_CTS(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-cts") + self.etc_cts_generated_code="/tmp/cts-generated" + self.code_generator="scripts/formatChecker/generate.py" + self.python_engine="python3" + self._generate_ets_code_from_template() + self.test_root=self.etc_cts_generated_code + self.add_directory(self.etc_cts_generated_code, "ets", []) + + + def _generate_ets_code_from_template(self): + cmd_string = f"{self.python_engine} {self.code_generator} ./CTS {self.etc_cts_generated_code}" + args=shlex.split(cmd_string) + work_dir = os.path.join(self.test_root, "tests") + process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_dir, env=self.cmd_env) + + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Cannot generate CTS code from template : timeout") + + if process.returncode != 0: + raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) + - def _generate_ets_code_from_template(): - pass + def create_test(self, test_file, flags, is_ignored): + test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) + test.ignored = is_ignored + return test diff --git a/test/runner/starter.py b/test/runner/starter.py index 553faa5f3..599ee95fd 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -44,7 +44,7 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - parser.add_argument('--ets-cts', action='store_true', dest='ets_sts', + parser.add_argument('--ets-cts', action='store_true', dest='ets_cts', default=False, help='run test agaisnt ETS specification') parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', default=False, help='run test agaisnt ETS specification') diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 8ec62c3d5..e75bf4680 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -10,8 +10,16 @@ from test_base import Test class TestETS(Test): - def __init__(self, test_env, test_path, flags, test_id, update_expected): + def __init__(self, test_path, test_env, flags, test_id, update_expected): Test.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further - self.fail_kind = None \ No newline at end of file + self.fail_kind = None + + def run_compiler(self): + options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --output={} --opt-level=0 {self.test_path}" + self.test_env.compiler + pass + + def run_runtime(): + pass \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py index 80e421fa8..8933ad052 100644 --- a/test/runner/test_ets_cts.py +++ b/test/runner/test_ets_cts.py @@ -3,9 +3,9 @@ from os import path, makedirs from configuration_kind import ConfigurationKind from test_ets import TestETS from utils import purify - +# test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) class TestETS_CTS(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): + def __init__(self, test_path, test_env, flags, test_id=None, update_expected=False): TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) self.tmp_dir = path.join(path.sep, "tmp", "cts") makedirs(self.tmp_dir, exist_ok=True) -- Gitee From 6cfe453826db7cdf53ca3900a691c5827107fdec Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Wed, 19 Oct 2022 16:16:50 +0300 Subject: [PATCH 05/12] execution of CTS was debugged --- test/runner/runner_base.py | 1 + test/runner/runner_ets.py | 55 +++++++++++++++++------------- test/runner/runner_ets_cts.py | 5 +-- test/runner/test_ets.py | 64 +++++++++++++++++++++++++++++++---- test/runner/test_ets_cts.py | 2 +- 5 files changed, 93 insertions(+), 34 deletions(-) diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 506e7c46a..0aadc492a 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -180,6 +180,7 @@ class Runner: ) ))) + def create_test(self, test_file, flags, is_ignored): pass diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 0f028d6e3..11f7ef6e7 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -28,40 +28,47 @@ INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" class RunnerETS(Runner): - def __init__(self, args, name:str): + def __init__(self, args, name: str): Runner.__init__(self, args, name) - output_path = path.join(self.stdlib_path, self.stdlib_name) + self.stdlib_path = path.join(self.test_root, "stdlib") + self.stdlib_name = "etsstdlib.abc" + stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) self.cmd_env = environ.copy() - self.test_root=args.test_root - self.compiler=path.join(self.build_dir, "bin", "es2panda") + self.test_root = args.test_root + self.compiler = path.join(self.build_dir, "bin", "es2panda") self.runtime = path.join(self.build_dir, "bin", "ark") - self.runtime_args = f"--boot-panda={output_path} --load_runtime=ets" self.cmd_env = environ.copy() - self.stdlib_name="etsstdlib.abc" - self.bytecode_path=path.join("/tmp", "ets", "cts") - self.stdlib_path=path.join(self.test_root, "stdlib") - self.compiler_args = f"--stdlib={self.stdlib_path} --gen-stdlib=false --extension=ets --opt_level=0" + self.stdlib_name = "etsstdlib.abc" + self.stdlib_path = path.join(self.test_root, "stdlib") - stdlib_file = Path(output_path) + + stdlib_file = Path(stdlib_output_path) if not stdlib_file.exists(): - self._generate_ets_stdlib(self.stdlib_path, output_path) + self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) self.test_env = TestEnv( args=args, cmd_env=self.cmd_env, - compiler=self.compiler, + es2panda =self.compiler, runtime=self.runtime, - compiler_args = self.compiler_args, - runtime_args=self.runtime_args, - stdlib_path=self.stdlib_path, - bytecode_path=self.bytecode_path + runtime_args="", + arkaout="", + aot_args="", + ark_quick="", + quick_args="", + conf_kind="", + cmd_prefix="" ) - def _generate_ets_stdlib(self, stdlib_path : str, output_path : str): - dummy_test=path.join(self.test_root, "stdlib/std/core/Boolean.ets") - options=f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" - cmd_string=f"{self.compiler} {options}" - args=shlex.split(cmd_string) - process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) + self.test_env.stdlib_path = self.stdlib_path + self.test_env.stdlib_output_path = stdlib_output_path + + def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): + dummy_test = path.join(self.test_root, "stdlib/std/core/Boolean.ets") + options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" + cmd_string = f"{self.compiler} {options}" + args = shlex.split(cmd_string) + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) try: out, err = process.communicate(timeout=600) @@ -70,5 +77,5 @@ class RunnerETS(Runner): raise Exception("Cannot generate ETS stdlib : timeout") if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - + raise Exception("Generating ETS stdlib fail %d" % + (process.returncode)) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py index aba767ab7..0b782f0c3 100644 --- a/test/runner/runner_ets_cts.py +++ b/test/runner/runner_ets_cts.py @@ -3,7 +3,7 @@ import shlex import subprocess from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_cts import TestETS_CTS +from test_ets import TestETS class RunnerETS_CTS(RunnerETS): @@ -13,6 +13,7 @@ class RunnerETS_CTS(RunnerETS): self.code_generator="scripts/formatChecker/generate.py" self.python_engine="python3" self._generate_ets_code_from_template() + self.test_root_backup = self.test_root self.test_root=self.etc_cts_generated_code self.add_directory(self.etc_cts_generated_code, "ets", []) @@ -34,7 +35,7 @@ class RunnerETS_CTS(RunnerETS): def create_test(self, test_file, flags, is_ignored): - test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) + test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) test.ignored = is_ignored return test diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index e75bf4680..b2568ec91 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -2,6 +2,7 @@ import subprocess import sys from os import path, remove from typing import List +import shlex from configuration_kind import ConfigurationKind from fail_kind import FailKind @@ -10,16 +11,65 @@ from test_base import Test class TestETS(Test): - def __init__(self, test_path, test_env, flags, test_id, update_expected): + def __init__(self, test_env, test_path, flags, test_id, update_expected=False): Test.__init__(self, test_env, test_path, flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None + self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" + self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" + self.common_ets_runtime_options = test_env.runtime_args + self.ets_main_entry_point="ETSGLOBAL::main" + self.bytecode_path = path.join("/tmp", "ets", "cts") - def run_compiler(self): - options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --output={} --opt-level=0 {self.test_path}" - self.test_env.compiler - pass + def do_run(self): + test_basename = path.basename(self.path) + is_negative_compilation = False + is_negative_runtime = False + if test_basename.startswith("n."): + is_negative_compilation = True + if test_basename.startswith("n.e."): + is_negative_runtime = True - def run_runtime(): - pass \ No newline at end of file + compiled_file, rc = self._run_compiler(is_negative_compilation) + if rc > 0 : + return ; + + if not is_negative_compilation: + self._run_runtime(compiled_file, is_negative_runtime) + + def _run_compiler(self, is_negative = False) -> str: + test_basename = path.basename(self.path) + test_abc = f"{test_basename}.abc" + output_path = path.join(self.bytecode_path, "abc", test_abc) + ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" + args = shlex.split(compiler_cmd_string) + rc = self._run_process_from_python(args, is_negative) + return (output_path,rc) + + + def _run_runtime(self, abc_file:str, is_negative:bool): + runtime_cmd_string = f"{self.common_ets_runtime_options} {abc_file} {self.ets_main_entry_point}" + args = shlex.split(runtime_cmd_string) + runtime_rc = self._run_process_from_python(args, is_negative) + return runtime_rc + + def _run_process_from_python(self, args:list, is_negative:bool)->None: + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + + try: + out, err = process.communicate(timeout=600) + except subprocess.TimeoutExpired: + process.kill() + raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) + + if process.returncode != 0: + if is_negative: + return 0 + else: + return 1 + + + + \ No newline at end of file diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py index 8933ad052..ff1e7f510 100644 --- a/test/runner/test_ets_cts.py +++ b/test/runner/test_ets_cts.py @@ -5,7 +5,7 @@ from test_ets import TestETS from utils import purify # test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) class TestETS_CTS(TestETS): - def __init__(self, test_path, test_env, flags, test_id=None, update_expected=False): + def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) self.tmp_dir = path.join(path.sep, "tmp", "cts") makedirs(self.tmp_dir, exist_ok=True) -- Gitee From 64ba088006fc1cca55980a950a8871b0d8c3bb44 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Wed, 19 Oct 2022 19:30:16 +0300 Subject: [PATCH 06/12] hot fixes for execution of CTS was debugged --- test/runner/test_ets.py | 52 +++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index b2568ec91..4c6353437 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -1,6 +1,6 @@ import subprocess import sys -from os import path, remove +from os import path, makedirs from typing import List import shlex @@ -18,9 +18,9 @@ class TestETS(Test): self.fail_kind = None self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" - self.common_ets_runtime_options = test_env.runtime_args self.ets_main_entry_point="ETSGLOBAL::main" - self.bytecode_path = path.join("/tmp", "ets", "cts") + self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") + makedirs(self.bytecode_path, exist_ok=True) def do_run(self): test_basename = path.basename(self.path) @@ -33,42 +33,54 @@ class TestETS(Test): compiled_file, rc = self._run_compiler(is_negative_compilation) if rc > 0 : - return ; + return self; + else: + print("Compilation Passed") if not is_negative_compilation: - self._run_runtime(compiled_file, is_negative_runtime) + runtime_rc = self._run_runtime(compiled_file, is_negative_runtime) + if runtime_rc == 0: + print ("Runtime Passed") + else: + print ("Runtime Failed") + return self - def _run_compiler(self, is_negative = False) -> str: + def _run_compiler(self, is_negative:bool) -> str: test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" - output_path = path.join(self.bytecode_path, "abc", test_abc) + output_path = path.join(self.bytecode_path, test_abc) ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - rc = self._run_process_from_python(args, is_negative) - return (output_path,rc) + compile_rc = self._run_process_from_python(args) + if is_negative and compile_rc != 0: + return (output_path, 0) + elif is_negative and compile_rc == 0: + return (output_path, 1) + else: + return(output_path, 0) def _run_runtime(self, abc_file:str, is_negative:bool): - runtime_cmd_string = f"{self.common_ets_runtime_options} {abc_file} {self.ets_main_entry_point}" + runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - runtime_rc = self._run_process_from_python(args, is_negative) - return runtime_rc + runtime_rc = self._run_process_from_python(args) + if is_negative and runtime_rc != 0: + return 0 + elif is_negative and runtime_rc == 0: + return 1 + else: + return 0 - def _run_process_from_python(self, args:list, is_negative:bool)->None: + def _run_process_from_python(self, args:list): process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - out, err = process.communicate(timeout=600) + out, err = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - if process.returncode != 0: - if is_negative: - return 0 - else: - return 1 + return process.returncode != 0 -- Gitee From c2078c10705b89ddf1dd86dfa0b12921c6453cdc Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 11:27:12 +0300 Subject: [PATCH 07/12] update for report generating --- test/runner/test_ets.py | 45 ++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 4c6353437..2a3a52bdf 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -31,18 +31,13 @@ class TestETS(Test): if test_basename.startswith("n.e."): is_negative_runtime = True - compiled_file, rc = self._run_compiler(is_negative_compilation) - if rc > 0 : + self.compile_report, compiled_file = self._run_compiler(is_negative_compilation) + if self.compile_report.return_code > 0 : return self; - else: - print("Compilation Passed") if not is_negative_compilation: - runtime_rc = self._run_runtime(compiled_file, is_negative_runtime) - if runtime_rc == 0: - print ("Runtime Passed") - else: - print ("Runtime Failed") + self.runtime_report = self._run_runtime(compiled_file, is_negative_runtime) + return self def _run_compiler(self, is_negative:bool) -> str: @@ -52,35 +47,47 @@ class TestETS(Test): ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - compile_rc = self._run_process_from_python(args) + output_content, error_content, compile_rc = self._run_process_from_python(args) + compile_report = TestReport( + output=output_content, + error = error_content, + return_code = 0 + ) if is_negative and compile_rc != 0: - return (output_path, 0) + return (compile_report, output_path) elif is_negative and compile_rc == 0: - return (output_path, 1) + compile_report.return_code = 1 + return (compile_report, output_path) else: - return(output_path, 0) + return(compile_report, output_path) def _run_runtime(self, abc_file:str, is_negative:bool): runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - runtime_rc = self._run_process_from_python(args) + output_content, error_content, runtime_rc = self._run_process_from_python(args) + runtime_report = TestReport( + output=output_content, + error = error_content, + return_code = 0 + ) if is_negative and runtime_rc != 0: - return 0 + return runtime_report elif is_negative and runtime_rc == 0: - return 1 + runtime_report.return_code = 1 + return runtime_report else: - return 0 + return runtime_report def _run_process_from_python(self, args:list): process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: - out, err = process.communicate(timeout=60) + output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - return process.returncode != 0 + return (output_content, err_content,process.returncode) -- Gitee From b3c23c9f68e1426514a9d55e9139ae6d50311957 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 12:44:48 +0300 Subject: [PATCH 08/12] report generating was debugged --- test/runner/runner_ets.py | 5 +- test/runner/runner_ets_stdlib.py | 5 +- .../{runner_js.py => runner_file_based.py} | 2 +- test/runner/runner_js_hermes.py | 6 +- test/runner/runner_js_parser.py | 4 +- test/runner/runner_js_test262.py | 6 +- test/runner/test_ets.py | 80 +++++++++++-------- test/runner/test_ets_cts.py | 13 --- test/runner/test_ets_stdlib.py | 13 --- 9 files changed, 61 insertions(+), 73 deletions(-) rename test/runner/{runner_js.py => runner_file_based.py} (99%) delete mode 100644 test/runner/test_ets_cts.py delete mode 100644 test/runner/test_ets_stdlib.py diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 11f7ef6e7..f0720a5f9 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -13,6 +13,7 @@ from fail_kind import FailKind from params import TestEnv from report_format import ReportFormat from runner_base import Runner +from runner_file_based import RunnerFileBased from utils import write_2_file INDEX_TITLE = "${Title}" @@ -27,9 +28,9 @@ INDEX_TEST_NAME = "${TestName}" INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" -class RunnerETS(Runner): +class RunnerETS(RunnerFileBased): def __init__(self, args, name: str): - Runner.__init__(self, args, name) + RunnerFileBased.__init__(self, args, name) self.stdlib_path = path.join(self.test_root, "stdlib") self.stdlib_name = "etsstdlib.abc" stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index cf801d317..c8ebbd373 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,15 +1,12 @@ from runner_base import correct_path, get_test_id from runner_ets import RunnerETS -from test_ets_stdlib import TestETS_STDLIB class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): - RunnerETS.__init__(self, args, "ets-cts") - - def _generate_ets_code_from_template(): pass + \ No newline at end of file diff --git a/test/runner/runner_js.py b/test/runner/runner_file_based.py similarity index 99% rename from test/runner/runner_js.py rename to test/runner/runner_file_based.py index 8cefcaef6..f91eece86 100644 --- a/test/runner/runner_js.py +++ b/test/runner/runner_file_based.py @@ -24,7 +24,7 @@ INDEX_TEST_NAME = "${TestName}" INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" -class RunnerJS(Runner): +class RunnerFileBased(Runner): def __init__(self, args, name): Runner.__init__(self, args, name) self.cmd_env = environ.copy() diff --git a/test/runner/runner_js_hermes.py b/test/runner/runner_js_hermes.py index 79cf36021..9c20648b9 100644 --- a/test/runner/runner_js_hermes.py +++ b/test/runner/runner_js_hermes.py @@ -1,12 +1,12 @@ from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_hermes import TestJSHermes from util_hermes import UtilHermes -class RunnerJSHermes(RunnerJS): +class RunnerJSHermes(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "hermes") + RunnerFileBased.__init__(self, args, "hermes") self.collect_excluded_test_lists() self.collect_ignored_test_lists() diff --git a/test/runner/runner_js_parser.py b/test/runner/runner_js_parser.py index 2a134867c..d10fe8bef 100644 --- a/test/runner/runner_js_parser.py +++ b/test/runner/runner_js_parser.py @@ -1,11 +1,11 @@ from os import path from runner_base import get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_parser import TestJSParser -class RunnerJSParser(RunnerJS): +class RunnerJSParser(RunnerFileBased): def __init__(self, args): super(RunnerJSParser, self).__init__(args, "parser-js") diff --git a/test/runner/runner_js_test262.py b/test/runner/runner_js_test262.py index 21a880723..d16af1f07 100644 --- a/test/runner/runner_js_test262.py +++ b/test/runner/runner_js_test262.py @@ -1,14 +1,14 @@ from os import path from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_test262 import TestJSTest262 from util_test262 import UtilTest262 -class RunnerJSTest262(RunnerJS): +class RunnerJSTest262(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "test262-ark") + RunnerFileBased.__init__(self, args, "test262-ark") self.ignored_name_prefix = "test262" self.collect_excluded_test_lists(test_name=self.ignored_name_prefix) diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 2a3a52bdf..22aee1fa9 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -12,13 +12,14 @@ from test_base import Test class TestETS(Test): def __init__(self, test_env, test_path, flags, test_id, update_expected=False): - Test.__init__(self, test_env, test_path, flags, test_id, update_expected) + Test.__init__(self, test_env, test_path, + flags, test_id, update_expected) # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" - self.ets_main_entry_point="ETSGLOBAL::main" + self.ets_main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") makedirs(self.bytecode_path, exist_ok=True) @@ -31,45 +32,62 @@ class TestETS(Test): if test_basename.startswith("n.e."): is_negative_runtime = True - self.compile_report, compiled_file = self._run_compiler(is_negative_compilation) - if self.compile_report.return_code > 0 : - return self; + self.report, compiled_file = self._run_compiler(is_negative_compilation) + if self.report.return_code > 0: + self.passed=False + self.fail_kind= FailKind.ES2PANDA_FAIL + return self + elif self.report.return_code == 0 and is_negative_compilation: + self.passed = True + return self + else: + pass - if not is_negative_compilation: - self.runtime_report = self._run_runtime(compiled_file, is_negative_runtime) + if not is_negative_compilation: + self.report = self._run_runtime(compiled_file, is_negative_runtime) + if self.report.return_code > 0: + self.passed = False + self.fail_kind = FailKind.RUNTIME_FAIL + else: + self.passed = True return self - def _run_compiler(self, is_negative:bool) -> str: + def _run_compiler(self, is_negative: bool) -> str: test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - ets_compiler_options=f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - output_content, error_content, compile_rc = self._run_process_from_python(args) - compile_report = TestReport( - output=output_content, - error = error_content, - return_code = 0 + output_content, error_content, compile_rc = self._run_process_from_python( + args) + compile_report_succes = TestReport( + output=output_content.decode('UTF-8'), + error=error_content.decode('UTF-8'), + return_code=0 + ) + compile_report_fail = TestReport( + output=output_content.decode('UTF-8'), + error=error_content.decode('UTF-8'), + return_code=1 ) if is_negative and compile_rc != 0: - return (compile_report, output_path) + return (compile_report_succes, output_path) elif is_negative and compile_rc == 0: - compile_report.return_code = 1 - return (compile_report, output_path) + return (compile_report_fail, output_path) else: - return(compile_report, output_path) - + return (compile_report_succes, output_path) - def _run_runtime(self, abc_file:str, is_negative:bool): - runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" + def _run_runtime(self, abc_file: str, is_negative: bool): + runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - output_content, error_content, runtime_rc = self._run_process_from_python(args) + output_content, error_content, runtime_rc = self._run_process_from_python( + args) runtime_report = TestReport( output=output_content, - error = error_content, - return_code = 0 + error=error_content, + return_code=0 ) if is_negative and runtime_rc != 0: return runtime_report @@ -79,16 +97,14 @@ class TestETS(Test): else: return runtime_report - def _run_process_from_python(self, args:list): - process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + def _run_process_from_python(self, args: list): + process = subprocess.Popen( + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() - raise Exception("Hmm .... Task completed %s : timeout" % (self.test_id)) - - return (output_content, err_content,process.returncode) - - + raise Exception( + "Hmm .... Task completed %s : timeout" % (self.test_id)) - \ No newline at end of file + return (output_content, err_content, process.returncode) diff --git a/test/runner/test_ets_cts.py b/test/runner/test_ets_cts.py deleted file mode 100644 index ff1e7f510..000000000 --- a/test/runner/test_ets_cts.py +++ /dev/null @@ -1,13 +0,0 @@ -from os import path, makedirs - -from configuration_kind import ConfigurationKind -from test_ets import TestETS -from utils import purify -# test = TestETS_CTS(test_file, get_test_id(test_file, self.test_root)) -class TestETS_CTS(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): - TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) - self.tmp_dir = path.join(path.sep, "tmp", "cts") - makedirs(self.tmp_dir, exist_ok=True) - self.util = self.test_env.util - diff --git a/test/runner/test_ets_stdlib.py b/test/runner/test_ets_stdlib.py deleted file mode 100644 index dd1b3fb07..000000000 --- a/test/runner/test_ets_stdlib.py +++ /dev/null @@ -1,13 +0,0 @@ -from os import path, makedirs - -from configuration_kind import ConfigurationKind -from test_ets import TestETS -from utils import purify - -class TestETS_STDLIB(TestETS): - def __init__(self, test_env, test_path, flags, test_id=None, update_expected=False): - TestETS.__init__(self, test_env, test_path, flags, test_id, update_expected) - self.tmp_dir = path.join(path.sep, "tmp", "cts") - makedirs(self.tmp_dir, exist_ok=True) - self.util = self.test_env.util - -- Gitee From 0c113fbe615c2e6f2092276eed553d172751af86 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 18:09:18 +0300 Subject: [PATCH 09/12] update for stdlib --- test/runner/runner.py | 4 -- test/runner/runner_ets_cts.py | 44 -------------- test/runner/runner_ets_stdlib.py | 15 ++++- test/runner/test_ets.py | 98 +++++++++++++++----------------- 4 files changed, 58 insertions(+), 103 deletions(-) delete mode 100644 test/runner/runner_ets_cts.py diff --git a/test/runner/runner.py b/test/runner/runner.py index 74ca8960a..995ced502 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -4,7 +4,6 @@ from dotenv import load_dotenv from runner_js_hermes import RunnerJSHermes from runner_js_parser import RunnerJSParser from runner_js_test262 import RunnerJSTest262 -from runner_ets_cts import RunnerETS_CTS from runner_ets_stdlib import RunnerETS_STDLIB from starter import get_args @@ -29,9 +28,6 @@ def main(): if args.hermes: runners.append(RunnerJSHermes(args)) - if args.ets_cts: - runners.append(RunnerETS_CTS(args)) - if args.ets_stdlib: runners.append(RunnerETS_STDLIB(args)) diff --git a/test/runner/runner_ets_cts.py b/test/runner/runner_ets_cts.py deleted file mode 100644 index 0b782f0c3..000000000 --- a/test/runner/runner_ets_cts.py +++ /dev/null @@ -1,44 +0,0 @@ -import os.path -import shlex -import subprocess -from runner_base import correct_path, get_test_id -from runner_ets import RunnerETS -from test_ets import TestETS - - -class RunnerETS_CTS(RunnerETS): - def __init__(self, args): - RunnerETS.__init__(self, args, "ets-cts") - self.etc_cts_generated_code="/tmp/cts-generated" - self.code_generator="scripts/formatChecker/generate.py" - self.python_engine="python3" - self._generate_ets_code_from_template() - self.test_root_backup = self.test_root - self.test_root=self.etc_cts_generated_code - self.add_directory(self.etc_cts_generated_code, "ets", []) - - - def _generate_ets_code_from_template(self): - cmd_string = f"{self.python_engine} {self.code_generator} ./CTS {self.etc_cts_generated_code}" - args=shlex.split(cmd_string) - work_dir = os.path.join(self.test_root, "tests") - process = subprocess.Popen(args,stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_dir, env=self.cmd_env) - - try: - out, err = process.communicate(timeout=600) - except subprocess.TimeoutExpired: - process.kill() - raise Exception("Cannot generate CTS code from template : timeout") - - if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) - - - def create_test(self, test_file, flags, is_ignored): - test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) - test.ignored = is_ignored - return test - - - - \ No newline at end of file diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index c8ebbd373..f2058b6ed 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -1,10 +1,21 @@ -from runner_base import correct_path, get_test_id +import shlex +import os +import subprocess from runner_ets import RunnerETS +from test_ets import TestETS +from runner_base import get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): - pass + RunnerETS.__init__(self, args, "ets-stdlib") + self.add_directory(self.root_test, "ets", []) + + + def create_test(self, test_file, flags, is_ignored): + test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) + test.ignored = is_ignored + return test diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 22aee1fa9..3362b6d8e 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -2,6 +2,7 @@ import subprocess import sys from os import path, makedirs from typing import List +from typing import Tuple import shlex from configuration_kind import ConfigurationKind @@ -20,82 +21,74 @@ class TestETS(Test): self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" self.ets_main_entry_point = "ETSGLOBAL::main" - self.bytecode_path = path.join("/tmp", "ets", "cts", "abc") + self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) def do_run(self): test_basename = path.basename(self.path) - is_negative_compilation = False - is_negative_runtime = False + is_negative_test = False if test_basename.startswith("n."): - is_negative_compilation = True - if test_basename.startswith("n.e."): - is_negative_runtime = True + is_negative_test = True - self.report, compiled_file = self._run_compiler(is_negative_compilation) - if self.report.return_code > 0: + report, compiled_file, fail_kind = self._run_compiler() + if report.return_code > 0: self.passed=False - self.fail_kind= FailKind.ES2PANDA_FAIL + self.fail_kind= fail_kind + self.report = report return self - elif self.report.return_code == 0 and is_negative_compilation: + + self.report = self._run_runtime(compiled_file, is_negative_test) + if self.report.return_code > 0 and is_negative_test: self.passed = True return self + elif self.report.return_code > 0: + self.passed = False + self.fail_kind = FailKind.RUNTIME_FAIL + return self else: - pass - - if not is_negative_compilation: - self.report = self._run_runtime(compiled_file, is_negative_runtime) - if self.report.return_code > 0: - self.passed = False - self.fail_kind = FailKind.RUNTIME_FAIL - else: - self.passed = True + self.passed = True + return self - return self - def _run_compiler(self, is_negative: bool) -> str: + def _run_compiler(self) -> Tuple[TestReport,str,FailKind]: + fail_kind = None test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) - output_content, error_content, compile_rc = self._run_process_from_python( - args) - compile_report_succes = TestReport( - output=output_content.decode('UTF-8'), - error=error_content.decode('UTF-8'), - return_code=0 - ) - compile_report_fail = TestReport( - output=output_content.decode('UTF-8'), - error=error_content.decode('UTF-8'), - return_code=1 + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + try: + stdout_content, stderr_content = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + fail_kind = FailKind.ES2PANDA_TIMEOUT + + report = TestReport( + output=stdout_content, + error=stderr_content, + return_code=process.returncode ) - if is_negative and compile_rc != 0: - return (compile_report_succes, output_path) - elif is_negative and compile_rc == 0: - return (compile_report_fail, output_path) - else: - return (compile_report_succes, output_path) + return (report, output_path, fail_kind) def _run_runtime(self, abc_file: str, is_negative: bool): + fail_kind = None runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) - output_content, error_content, runtime_rc = self._run_process_from_python( - args) + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + try: + stdout_content, stderr_content = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + fail_kind = FailKind.RUNTIME_TIMEOUT + runtime_report = TestReport( - output=output_content, - error=error_content, - return_code=0 + output=stdout_content, + error=stderr_content, + return_code=process.returncode ) - if is_negative and runtime_rc != 0: - return runtime_report - elif is_negative and runtime_rc == 0: - runtime_report.return_code = 1 - return runtime_report - else: - return runtime_report + return runtime_report, fail_kind def _run_process_from_python(self, args: list): process = subprocess.Popen( @@ -104,7 +97,6 @@ class TestETS(Test): output_content, err_content = process.communicate(timeout=60) except subprocess.TimeoutExpired: process.kill() - raise Exception( - "Hmm .... Task completed %s : timeout" % (self.test_id)) + fail_kind = FailKind.PROCESS_TIMEOUT - return (output_content, err_content, process.returncode) + return (output_content, err_content, process.returncode, fail_kind) -- Gitee From 4540301fa7425cf6d68a7cd43bf9553f6bdce42d Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Thu, 20 Oct 2022 18:24:27 +0300 Subject: [PATCH 10/12] update for ets-stdlib.not debugged --- test/runner/runner_ets.py | 4 +++- test/runner/starter.py | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index f0720a5f9..68c37d7b9 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -64,7 +64,7 @@ class RunnerETS(RunnerFileBased): self.test_env.stdlib_output_path = stdlib_output_path def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): - dummy_test = path.join(self.test_root, "stdlib/std/core/Boolean.ets") + dummy_test = path.join(self.test_root, "std/dummy.ets") options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" cmd_string = f"{self.compiler} {options}" args = shlex.split(cmd_string) @@ -77,6 +77,8 @@ class RunnerETS(RunnerFileBased): process.kill() raise Exception("Cannot generate ETS stdlib : timeout") + print("StdOut : " + out.decode('UTF-8')) + print("StdErr : " + err.decode('UTF-8')) if process.returncode != 0: raise Exception("Generating ETS stdlib fail %d" % (process.returncode)) diff --git a/test/runner/starter.py b/test/runner/starter.py index 599ee95fd..5853fc662 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -44,8 +44,6 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - parser.add_argument('--ets-cts', action='store_true', dest='ets_cts', - default=False, help='run test agaisnt ETS specification') parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', default=False, help='run test agaisnt ETS specification') parser.add_argument( -- Gitee From 792a7a3225ffae03373503a6d006b8a66b515cb0 Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Fri, 21 Oct 2022 16:05:59 +0300 Subject: [PATCH 11/12] support for stdlib test is ready for review 1 --- test/runner/runner_ets.py | 57 +++++++++++++++----------------- test/runner/runner_ets_stdlib.py | 3 +- test/runner/test_ets.py | 52 +++++++++++++---------------- 3 files changed, 51 insertions(+), 61 deletions(-) diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py index 68c37d7b9..68afaa391 100644 --- a/test/runner/runner_ets.py +++ b/test/runner/runner_ets.py @@ -1,11 +1,6 @@ -import subprocess -import sys -from datetime import datetime -from glob import glob -from os import path, environ, makedirs -from typing import List +from cProfile import run +from os import path, environ from pathlib import Path -import shlex from configuration_kind import ConfigurationKind @@ -39,19 +34,21 @@ class RunnerETS(RunnerFileBased): self.compiler = path.join(self.build_dir, "bin", "es2panda") self.runtime = path.join(self.build_dir, "bin", "ark") self.cmd_env = environ.copy() - self.stdlib_name = "etsstdlib.abc" - self.stdlib_path = path.join(self.test_root, "stdlib") + self.stdlib_path = path.join(self.build_dir, "etsstdlib.abc") + self.stdlib_src_path = path.join(self.test_root, "stdlib") + self._check_binary_artefacts() + stdlib_file = Path(stdlib_output_path) - if not stdlib_file.exists(): - self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) + # if not stdlib_file.exists(): + # self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) self.test_env = TestEnv( args=args, cmd_env=self.cmd_env, es2panda =self.compiler, runtime=self.runtime, - runtime_args="", + runtime_args=f"--boot-panda-files={self.stdlib_path} --load-runtimes=ets", arkaout="", aot_args="", ark_quick="", @@ -60,25 +57,23 @@ class RunnerETS(RunnerFileBased): cmd_prefix="" ) - self.test_env.stdlib_path = self.stdlib_path - self.test_env.stdlib_output_path = stdlib_output_path + self.test_env.es2panda_args= f"--stdlib={self.stdlib_src_path} --gen-stdlib=false --extension=ets --opt-level=0" + + + def _check_binary_artefacts(self) : + compiler_path_obj = Path(self.compiler) + runtime_path_obj = Path(self.runtime) + stdlib_path_obj = Path(self.stdlib_path) + stdlib_src_path_obj = Path(self.stdlib_src_path) + + if not compiler_path_obj.is_file(): + raise Exception("Hmm ... ETS Compiler was not found") - def _generate_ets_stdlib(self, stdlib_path: str, output_path: str): - dummy_test = path.join(self.test_root, "std/dummy.ets") - options = f"--stdlib={stdlib_path} --extension=ets --output={output_path} --opt-level=0 --gen-stdlib=true {dummy_test}" - cmd_string = f"{self.compiler} {options}" - args = shlex.split(cmd_string) - process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.cmd_env) + if not runtime_path_obj.is_file(): + raise Exception("Hmm ... ETS Runtime was not found") - try: - out, err = process.communicate(timeout=600) - except subprocess.TimeoutExpired: - process.kill() - raise Exception("Cannot generate ETS stdlib : timeout") + if not stdlib_path_obj.is_file(): + raise Exception("Hmm ... standart library abc file not found") - print("StdOut : " + out.decode('UTF-8')) - print("StdErr : " + err.decode('UTF-8')) - if process.returncode != 0: - raise Exception("Generating ETS stdlib fail %d" % - (process.returncode)) + if not stdlib_src_path_obj.is_dir(): + raise Exception("Hmm ... Source code of standart library was not found") \ No newline at end of file diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index f2058b6ed..6187e4782 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -9,7 +9,8 @@ from runner_base import get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-stdlib") - self.add_directory(self.root_test, "ets", []) + self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") + self.add_directory(self.stdlib_test_path, "ets", []) def create_test(self, test_file, flags, is_ignored): diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 3362b6d8e..172d4f1d4 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -18,8 +18,6 @@ class TestETS(Test): # If test fails it contains reason (of FailKind enum) of first failed step # It's supposed if the first step is failed then no step is executed further self.fail_kind = None - self.common_ets_compiler_options = f"--stdlib={self.test_env.stdlib_path} --gen-stdlib=false --extension=ets --opt-level=0" - self.common_runtime_options = f"--boot-panda={self.test_env.stdlib_output_path} --load_runtime=ets" self.ets_main_entry_point = "ETSGLOBAL::main" self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") makedirs(self.bytecode_path, exist_ok=True) @@ -31,19 +29,18 @@ class TestETS(Test): is_negative_test = True report, compiled_file, fail_kind = self._run_compiler() - if report.return_code > 0: - self.passed=False - self.fail_kind= fail_kind - self.report = report + self.fail_kind= fail_kind + self.report = report + if report.return_code != 0: + self.passed = False return self - self.report = self._run_runtime(compiled_file, is_negative_test) - if self.report.return_code > 0 and is_negative_test: + self.report, self.fail_kind = self._run_runtime(compiled_file) + if self.report.return_code != 0 and is_negative_test: self.passed = True return self - elif self.report.return_code > 0: + elif self.report.return_code != 0: self.passed = False - self.fail_kind = FailKind.RUNTIME_FAIL return self else: self.passed = True @@ -55,9 +52,10 @@ class TestETS(Test): test_basename = path.basename(self.path) test_abc = f"{test_basename}.abc" output_path = path.join(self.bytecode_path, test_abc) - ets_compiler_options = f"{self.common_ets_compiler_options} --output={output_path} {self.path}" + ets_compiler_options = f"{self.test_env.es2panda_args} --output={output_path} {self.path}" compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" args = shlex.split(compiler_cmd_string) + self.log_cmd(args) process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: stdout_content, stderr_content = process.communicate(timeout=60) @@ -65,17 +63,21 @@ class TestETS(Test): process.kill() fail_kind = FailKind.ES2PANDA_TIMEOUT + if process.returncode != 0: + fail_kind = FailKind.ES2PANDA_FAIL + report = TestReport( - output=stdout_content, - error=stderr_content, + output=stdout_content.decode('UTF-8'), + error=stderr_content.decode('UTF-8'), return_code=process.returncode ) return (report, output_path, fail_kind) - def _run_runtime(self, abc_file: str, is_negative: bool): + def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: fail_kind = None - runtime_cmd_string = f"{self.test_env.runtime} {self.common_runtime_options} {abc_file} {self.ets_main_entry_point}" + runtime_cmd_string = f"{self.test_env.runtime} {self.test_env.runtime_args} {abc_file} {self.ets_main_entry_point}" args = shlex.split(runtime_cmd_string) + self.log_cmd(args) process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) try: stdout_content, stderr_content = process.communicate(timeout=60) @@ -83,20 +85,12 @@ class TestETS(Test): process.kill() fail_kind = FailKind.RUNTIME_TIMEOUT + if process.returncode != 0: + fail_kind = FailKind.RUNTIME_FAIL + runtime_report = TestReport( - output=stdout_content, - error=stderr_content, + output=stdout_content.decode('UTF-8'), + error=stderr_content.decode('UTF-8'), return_code=process.returncode ) - return runtime_report, fail_kind - - def _run_process_from_python(self, args: list): - process = subprocess.Popen( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) - try: - output_content, err_content = process.communicate(timeout=60) - except subprocess.TimeoutExpired: - process.kill() - fail_kind = FailKind.PROCESS_TIMEOUT - - return (output_content, err_content, process.returncode, fail_kind) + return runtime_report, fail_kind \ No newline at end of file -- Gitee From 92e0c76d776cd9dcbaac4727a3d287b97d96efff Mon Sep 17 00:00:00 2001 From: aleksander gorlov Date: Tue, 25 Oct 2022 14:53:09 +0300 Subject: [PATCH 12/12] add excluded list processing --- test/ets-stdlib-excluded.txt | 3 +++ test/runner/runner_base.py | 1 + test/runner/runner_ets_stdlib.py | 5 ++++- test/runner/test_ets.py | 1 + 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 test/ets-stdlib-excluded.txt diff --git a/test/ets-stdlib-excluded.txt b/test/ets-stdlib-excluded.txt new file mode 100644 index 000000000..477b26172 --- /dev/null +++ b/test/ets-stdlib-excluded.txt @@ -0,0 +1,3 @@ +tests/stdlib/std/math/sqrt-negative-01.ets +tests/stdlib/std/math/sqrt-positive-03.ets +tests/stdlib/std/math/sqrt-positive-04.ets \ No newline at end of file diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 0aadc492a..a1c6fbbd2 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -133,6 +133,7 @@ class Runner: # Read excluded_lists and load list of excluded tests def load_excluded_tests(self): + print(self.excluded_lists) self.excluded_tests.update(self.load_tests_from_lists(self.excluded_lists)) self.excluded = len(self.excluded_tests) diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py index 6187e4782..97dd11067 100644 --- a/test/runner/runner_ets_stdlib.py +++ b/test/runner/runner_ets_stdlib.py @@ -3,14 +3,17 @@ import os import subprocess from runner_ets import RunnerETS from test_ets import TestETS -from runner_base import get_test_id +from runner_base import correct_path, get_test_id class RunnerETS_STDLIB(RunnerETS): def __init__(self, args): RunnerETS.__init__(self, args, "ets-stdlib") self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") + self.ignored_name_prefix = self.name + self.excluded_lists = [correct_path(self.list_root, f"{self.ignored_name_prefix}-excluded.txt")] self.add_directory(self.stdlib_test_path, "ets", []) + print (self.excluded_lists) def create_test(self, test_file, flags, is_ignored): diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py index 172d4f1d4..e7d666207 100644 --- a/test/runner/test_ets.py +++ b/test/runner/test_ets.py @@ -71,6 +71,7 @@ class TestETS(Test): error=stderr_content.decode('UTF-8'), return_code=process.returncode ) + return (report, output_path, fail_kind) def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: -- Gitee