diff --git a/test/ets-stdlib-excluded.txt b/test/ets-stdlib-excluded.txt new file mode 100644 index 0000000000000000000000000000000000000000..477b2617224b6755c1d21cccba9f320653e7c062 --- /dev/null +++ b/test/ets-stdlib-excluded.txt @@ -0,0 +1,3 @@ +tests/stdlib/std/math/sqrt-negative-01.ets +tests/stdlib/std/math/sqrt-positive-03.ets +tests/stdlib/std/math/sqrt-positive-04.ets \ No newline at end of file diff --git a/test/runner/runner.py b/test/runner/runner.py index 607e8cb259d7b94e607f7d7a262b8843ab18255c..995ced502111b5e486cd01a10c4f02917fd9f228 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -4,6 +4,7 @@ from dotenv import load_dotenv from runner_js_hermes import RunnerJSHermes from runner_js_parser import RunnerJSParser from runner_js_test262 import RunnerJSTest262 +from runner_ets_stdlib import RunnerETS_STDLIB from starter import get_args @@ -27,6 +28,9 @@ def main(): if args.hermes: runners.append(RunnerJSHermes(args)) + if args.ets_stdlib: + runners.append(RunnerETS_STDLIB(args)) + failed_tests = 0 for runner in runners: diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 506e7c46ac7eaf72b9c4a4f6622aa69b5a542034..a1c6fbbd239f6544c1d2a7a62771181fb05205b8 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -133,6 +133,7 @@ class Runner: # Read excluded_lists and load list of excluded tests def load_excluded_tests(self): + print(self.excluded_lists) self.excluded_tests.update(self.load_tests_from_lists(self.excluded_lists)) self.excluded = len(self.excluded_tests) @@ -180,6 +181,7 @@ class Runner: ) ))) + def create_test(self, test_file, flags, is_ignored): pass diff --git a/test/runner/runner_ets.py b/test/runner/runner_ets.py new file mode 100644 index 0000000000000000000000000000000000000000..68afaa391db4eeeb195f096e491ce80e5097c19e --- /dev/null +++ b/test/runner/runner_ets.py @@ -0,0 +1,79 @@ +from cProfile import run +from os import path, environ +from pathlib import Path + + +from configuration_kind import ConfigurationKind +from fail_kind import FailKind +from params import TestEnv +from report_format import ReportFormat +from runner_base import Runner +from runner_file_based import RunnerFileBased +from utils import write_2_file + +INDEX_TITLE = "${Title}" +INDEX_OPTIONS = "${Options}" +INDEX_TOTAL = "${Total}" +INDEX_PASSED = "${Passed}" +INDEX_FAILED = "${Failed}" +INDEX_IGNORED = "${Ignored}" +INDEX_EXCLUDED_LISTS = "${ExcludedThroughLists}" +INDEX_EXCLUDED_OTHER = "${ExcludedByOtherReasons}" +INDEX_TEST_NAME = "${TestName}" +INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" + + +class RunnerETS(RunnerFileBased): + def __init__(self, args, name: str): + RunnerFileBased.__init__(self, args, name) + self.stdlib_path = path.join(self.test_root, "stdlib") + self.stdlib_name = "etsstdlib.abc" + stdlib_output_path = path.join(self.stdlib_path, self.stdlib_name) + self.cmd_env = environ.copy() + self.test_root = args.test_root + self.compiler = path.join(self.build_dir, "bin", "es2panda") + self.runtime = path.join(self.build_dir, "bin", "ark") + self.cmd_env = environ.copy() + self.stdlib_path = path.join(self.build_dir, "etsstdlib.abc") + self.stdlib_src_path = path.join(self.test_root, "stdlib") + self._check_binary_artefacts() + + + + stdlib_file = Path(stdlib_output_path) + # if not stdlib_file.exists(): + # self._generate_ets_stdlib(self.stdlib_path, stdlib_output_path) + self.test_env = TestEnv( + args=args, + cmd_env=self.cmd_env, + es2panda =self.compiler, + runtime=self.runtime, + runtime_args=f"--boot-panda-files={self.stdlib_path} --load-runtimes=ets", + arkaout="", + aot_args="", + ark_quick="", + quick_args="", + conf_kind="", + cmd_prefix="" + ) + + self.test_env.es2panda_args= f"--stdlib={self.stdlib_src_path} --gen-stdlib=false --extension=ets --opt-level=0" + + + def _check_binary_artefacts(self) : + compiler_path_obj = Path(self.compiler) + runtime_path_obj = Path(self.runtime) + stdlib_path_obj = Path(self.stdlib_path) + stdlib_src_path_obj = Path(self.stdlib_src_path) + + if not compiler_path_obj.is_file(): + raise Exception("Hmm ... ETS Compiler was not found") + + if not runtime_path_obj.is_file(): + raise Exception("Hmm ... ETS Runtime was not found") + + if not stdlib_path_obj.is_file(): + raise Exception("Hmm ... standart library abc file not found") + + if not stdlib_src_path_obj.is_dir(): + raise Exception("Hmm ... Source code of standart library was not found") \ No newline at end of file diff --git a/test/runner/runner_ets_stdlib.py b/test/runner/runner_ets_stdlib.py new file mode 100644 index 0000000000000000000000000000000000000000..97dd110673391e472a3dcc37e877ac9564774f06 --- /dev/null +++ b/test/runner/runner_ets_stdlib.py @@ -0,0 +1,27 @@ +import shlex +import os +import subprocess +from runner_ets import RunnerETS +from test_ets import TestETS +from runner_base import correct_path, get_test_id + + +class RunnerETS_STDLIB(RunnerETS): + def __init__(self, args): + RunnerETS.__init__(self, args, "ets-stdlib") + self.stdlib_test_path = os.path.join(self.test_root, "tests/stdlib") + self.ignored_name_prefix = self.name + self.excluded_lists = [correct_path(self.list_root, f"{self.ignored_name_prefix}-excluded.txt")] + self.add_directory(self.stdlib_test_path, "ets", []) + print (self.excluded_lists) + + + def create_test(self, test_file, flags, is_ignored): + test = TestETS(self.test_env, test_file, flags, get_test_id(test_file, self.test_root)) + test.ignored = is_ignored + return test + + + + + \ No newline at end of file diff --git a/test/runner/runner_js.py b/test/runner/runner_file_based.py similarity index 99% rename from test/runner/runner_js.py rename to test/runner/runner_file_based.py index 8cefcaef6e7344032d71f8a5d7470d28bd93d712..f91eece8655828226136327efd5f3edbe1d62c07 100644 --- a/test/runner/runner_js.py +++ b/test/runner/runner_file_based.py @@ -24,7 +24,7 @@ INDEX_TEST_NAME = "${TestName}" INDEX_FAILED_TESTS_LIST = "${FailedTestsList}" -class RunnerJS(Runner): +class RunnerFileBased(Runner): def __init__(self, args, name): Runner.__init__(self, args, name) self.cmd_env = environ.copy() diff --git a/test/runner/runner_js_hermes.py b/test/runner/runner_js_hermes.py index 79cf36021cc9df70ccbad6336c7d216c8e78be7d..9c20648b9dd99e78745f0cb7bde8833b80858d20 100644 --- a/test/runner/runner_js_hermes.py +++ b/test/runner/runner_js_hermes.py @@ -1,12 +1,12 @@ from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_hermes import TestJSHermes from util_hermes import UtilHermes -class RunnerJSHermes(RunnerJS): +class RunnerJSHermes(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "hermes") + RunnerFileBased.__init__(self, args, "hermes") self.collect_excluded_test_lists() self.collect_ignored_test_lists() diff --git a/test/runner/runner_js_parser.py b/test/runner/runner_js_parser.py index 2a134867c091804abb6c883bdff5ce08782d7f54..d10fe8befaf6e869635fd20d66a4c2bbfdc88dcf 100644 --- a/test/runner/runner_js_parser.py +++ b/test/runner/runner_js_parser.py @@ -1,11 +1,11 @@ from os import path from runner_base import get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_parser import TestJSParser -class RunnerJSParser(RunnerJS): +class RunnerJSParser(RunnerFileBased): def __init__(self, args): super(RunnerJSParser, self).__init__(args, "parser-js") diff --git a/test/runner/runner_js_test262.py b/test/runner/runner_js_test262.py index 21a8807233c9f6e6b2b5c99fce3e99375d3d406e..d16af1f0741a2e3944db271a7121e46b7f33dd7d 100644 --- a/test/runner/runner_js_test262.py +++ b/test/runner/runner_js_test262.py @@ -1,14 +1,14 @@ from os import path from runner_base import correct_path, get_test_id -from runner_js import RunnerJS +from runner_file_based import RunnerFileBased from test_js_test262 import TestJSTest262 from util_test262 import UtilTest262 -class RunnerJSTest262(RunnerJS): +class RunnerJSTest262(RunnerFileBased): def __init__(self, args): - RunnerJS.__init__(self, args, "test262-ark") + RunnerFileBased.__init__(self, args, "test262-ark") self.ignored_name_prefix = "test262" self.collect_excluded_test_lists(test_name=self.ignored_name_prefix) diff --git a/test/runner/starter.py b/test/runner/starter.py index a1f1792f1637e4c3a3060aea093b0b10793104f2..5853fc662d5ee483c644776485ea6c9d6e4a9827 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -21,7 +21,8 @@ def is_file(parser, arg): def check_timeout(value): ivalue = int(value) if ivalue <= 0: - raise argparse.ArgumentTypeError(f"{value} is an invalid timeout value") + raise argparse.ArgumentTypeError( + f"{value} is an invalid timeout value") return ivalue @@ -43,7 +44,8 @@ def get_args(): parser.add_argument( '--tsc', action='store_true', dest='tsc', default=False, help='run tsc tests') - + parser.add_argument('--ets-stdlib', action='store_true', dest='ets_stdlib', + default=False, help='run test agaisnt ETS specification') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), help='directory with test file. If not set the module directory is used') diff --git a/test/runner/test_ets.py b/test/runner/test_ets.py new file mode 100644 index 0000000000000000000000000000000000000000..e7d666207c58b559e64dc61381e4c027bd2bec32 --- /dev/null +++ b/test/runner/test_ets.py @@ -0,0 +1,97 @@ +import subprocess +import sys +from os import path, makedirs +from typing import List +from typing import Tuple +import shlex + +from configuration_kind import ConfigurationKind +from fail_kind import FailKind +from params import Params, TestReport +from test_base import Test + + +class TestETS(Test): + def __init__(self, test_env, test_path, flags, test_id, update_expected=False): + Test.__init__(self, test_env, test_path, + flags, test_id, update_expected) + # If test fails it contains reason (of FailKind enum) of first failed step + # It's supposed if the first step is failed then no step is executed further + self.fail_kind = None + self.ets_main_entry_point = "ETSGLOBAL::main" + self.bytecode_path = path.join("/tmp", "ets", "stdlib", "abc") + makedirs(self.bytecode_path, exist_ok=True) + + def do_run(self): + test_basename = path.basename(self.path) + is_negative_test = False + if test_basename.startswith("n."): + is_negative_test = True + + report, compiled_file, fail_kind = self._run_compiler() + self.fail_kind= fail_kind + self.report = report + if report.return_code != 0: + self.passed = False + return self + + self.report, self.fail_kind = self._run_runtime(compiled_file) + if self.report.return_code != 0 and is_negative_test: + self.passed = True + return self + elif self.report.return_code != 0: + self.passed = False + return self + else: + self.passed = True + return self + + + def _run_compiler(self) -> Tuple[TestReport,str,FailKind]: + fail_kind = None + test_basename = path.basename(self.path) + test_abc = f"{test_basename}.abc" + output_path = path.join(self.bytecode_path, test_abc) + ets_compiler_options = f"{self.test_env.es2panda_args} --output={output_path} {self.path}" + compiler_cmd_string = f"{self.test_env.es2panda} {ets_compiler_options}" + args = shlex.split(compiler_cmd_string) + self.log_cmd(args) + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + try: + stdout_content, stderr_content = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + fail_kind = FailKind.ES2PANDA_TIMEOUT + + if process.returncode != 0: + fail_kind = FailKind.ES2PANDA_FAIL + + report = TestReport( + output=stdout_content.decode('UTF-8'), + error=stderr_content.decode('UTF-8'), + return_code=process.returncode + ) + + return (report, output_path, fail_kind) + + def _run_runtime(self, abc_file: str) -> Tuple[TestReport, FailKind]: + fail_kind = None + runtime_cmd_string = f"{self.test_env.runtime} {self.test_env.runtime_args} {abc_file} {self.ets_main_entry_point}" + args = shlex.split(runtime_cmd_string) + self.log_cmd(args) + process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=self.test_env.cmd_env) + try: + stdout_content, stderr_content = process.communicate(timeout=60) + except subprocess.TimeoutExpired: + process.kill() + fail_kind = FailKind.RUNTIME_TIMEOUT + + if process.returncode != 0: + fail_kind = FailKind.RUNTIME_FAIL + + runtime_report = TestReport( + output=stdout_content.decode('UTF-8'), + error=stderr_content.decode('UTF-8'), + return_code=process.returncode + ) + return runtime_report, fail_kind \ No newline at end of file