From 22638bfecac5e0c6d93c5513a3d8778a1e659280 Mon Sep 17 00:00:00 2001 From: Viktoria Shirunova Date: Wed, 12 Oct 2022 21:33:38 +0300 Subject: [PATCH] es2panda/js test runner: convert test suites to plugins. Keep test lists at the corresponding plugin by default Signed-off-by: Viktoria Shirunova --- .../hermes}/hermes-excluded-AOT-FULL.txt | 0 .../plugins/hermes}/hermes-excluded.txt | 0 .../hermes}/hermes-ignored-AOT-FULL.txt | 0 .../plugins/hermes}/hermes-ignored-AOT.txt | 0 .../plugins/hermes}/hermes-ignored-JIT.txt | 0 .../{ => plugins/hermes}/hermes_harness.py | 0 .../{ => plugins/hermes}/runner_js_hermes.py | 7 +- .../{ => plugins/hermes}/test_js_hermes.py | 0 .../{ => plugins/hermes}/util_hermes.py | 0 .../{ => plugins/parser}/runner_js_parser.py | 5 +- .../{ => plugins/parser}/test_js_parser.py | 0 .../test262}/runner_js_test262.py | 7 +- .../plugins/test262}/test262-excluded.txt | 0 .../test262-flaky-ignored-AOT-FULL.txt | 0 .../test262}/test262-flaky-ignored-AOT.txt | 0 .../test262}/test262-flaky-ignored-INT.txt | 0 .../test262}/test262-flaky-ignored-IRTOC.txt | 0 .../test262}/test262-flaky-ignored-JIT.txt | 0 .../test262}/test262-ignored-AOT-FULL.txt | 0 .../plugins/test262}/test262-ignored-AOT.txt | 0 .../plugins/test262}/test262-ignored-INT.txt | 0 .../test262}/test262-ignored-IRTOC.txt | 0 .../plugins/test262}/test262-ignored-JIT.txt | 2 + .../plugins/test262}/test262-ignored.txt | 0 .../test262}/test262-long-excluded.txt | 0 .../{ => plugins/test262}/test262harness.js | 0 .../plugins/test262}/test262skiplist-bco.txt | 0 .../test262skiplist-flaky-ARM32INT.txt | 0 .../{ => plugins/test262}/test_js_test262.py | 0 .../{ => plugins/test262}/util_test262.py | 0 test/runner/plugins_registry.py | 100 ++++++++++++++++++ test/runner/runner.py | 27 ++--- test/runner/runner_base.py | 2 +- test/runner/starter.py | 7 +- 34 files changed, 135 insertions(+), 22 deletions(-) rename test/{ => runner/plugins/hermes}/hermes-excluded-AOT-FULL.txt (100%) rename test/{ => runner/plugins/hermes}/hermes-excluded.txt (100%) rename test/{ => runner/plugins/hermes}/hermes-ignored-AOT-FULL.txt (100%) rename test/{ => runner/plugins/hermes}/hermes-ignored-AOT.txt (100%) rename test/{ => runner/plugins/hermes}/hermes-ignored-JIT.txt (100%) rename test/runner/{ => plugins/hermes}/hermes_harness.py (100%) rename test/runner/{ => plugins/hermes}/runner_js_hermes.py (82%) rename test/runner/{ => plugins/hermes}/test_js_hermes.py (100%) rename test/runner/{ => plugins/hermes}/util_hermes.py (100%) rename test/runner/{ => plugins/parser}/runner_js_parser.py (76%) rename test/runner/{ => plugins/parser}/test_js_parser.py (100%) rename test/runner/{ => plugins/test262}/runner_js_test262.py (81%) rename test/{ => runner/plugins/test262}/test262-excluded.txt (100%) rename test/{ => runner/plugins/test262}/test262-flaky-ignored-AOT-FULL.txt (100%) rename test/{ => runner/plugins/test262}/test262-flaky-ignored-AOT.txt (100%) rename test/{ => runner/plugins/test262}/test262-flaky-ignored-INT.txt (100%) rename test/{ => runner/plugins/test262}/test262-flaky-ignored-IRTOC.txt (100%) rename test/{ => runner/plugins/test262}/test262-flaky-ignored-JIT.txt (100%) rename test/{ => runner/plugins/test262}/test262-ignored-AOT-FULL.txt (100%) rename test/{ => runner/plugins/test262}/test262-ignored-AOT.txt (100%) rename test/{ => runner/plugins/test262}/test262-ignored-INT.txt (100%) rename test/{ => runner/plugins/test262}/test262-ignored-IRTOC.txt (100%) rename test/{ => runner/plugins/test262}/test262-ignored-JIT.txt (98%) rename test/{ => runner/plugins/test262}/test262-ignored.txt (100%) rename test/{ => runner/plugins/test262}/test262-long-excluded.txt (100%) rename test/runner/{ => plugins/test262}/test262harness.js (100%) rename test/{ => runner/plugins/test262}/test262skiplist-bco.txt (100%) rename test/{ => runner/plugins/test262}/test262skiplist-flaky-ARM32INT.txt (100%) rename test/runner/{ => plugins/test262}/test_js_test262.py (100%) rename test/runner/{ => plugins/test262}/util_test262.py (100%) create mode 100644 test/runner/plugins_registry.py diff --git a/test/hermes-excluded-AOT-FULL.txt b/test/runner/plugins/hermes/hermes-excluded-AOT-FULL.txt similarity index 100% rename from test/hermes-excluded-AOT-FULL.txt rename to test/runner/plugins/hermes/hermes-excluded-AOT-FULL.txt diff --git a/test/hermes-excluded.txt b/test/runner/plugins/hermes/hermes-excluded.txt similarity index 100% rename from test/hermes-excluded.txt rename to test/runner/plugins/hermes/hermes-excluded.txt diff --git a/test/hermes-ignored-AOT-FULL.txt b/test/runner/plugins/hermes/hermes-ignored-AOT-FULL.txt similarity index 100% rename from test/hermes-ignored-AOT-FULL.txt rename to test/runner/plugins/hermes/hermes-ignored-AOT-FULL.txt diff --git a/test/hermes-ignored-AOT.txt b/test/runner/plugins/hermes/hermes-ignored-AOT.txt similarity index 100% rename from test/hermes-ignored-AOT.txt rename to test/runner/plugins/hermes/hermes-ignored-AOT.txt diff --git a/test/hermes-ignored-JIT.txt b/test/runner/plugins/hermes/hermes-ignored-JIT.txt similarity index 100% rename from test/hermes-ignored-JIT.txt rename to test/runner/plugins/hermes/hermes-ignored-JIT.txt diff --git a/test/runner/hermes_harness.py b/test/runner/plugins/hermes/hermes_harness.py similarity index 100% rename from test/runner/hermes_harness.py rename to test/runner/plugins/hermes/hermes_harness.py diff --git a/test/runner/runner_js_hermes.py b/test/runner/plugins/hermes/runner_js_hermes.py similarity index 82% rename from test/runner/runner_js_hermes.py rename to test/runner/plugins/hermes/runner_js_hermes.py index 79cf36021..16b0e5581 100644 --- a/test/runner/runner_js_hermes.py +++ b/test/runner/plugins/hermes/runner_js_hermes.py @@ -1,13 +1,16 @@ +from os import path + from runner_base import correct_path, get_test_id from runner_js import RunnerJS -from test_js_hermes import TestJSHermes -from util_hermes import UtilHermes +from plugins.hermes.test_js_hermes import TestJSHermes +from plugins.hermes.util_hermes import UtilHermes class RunnerJSHermes(RunnerJS): def __init__(self, args): RunnerJS.__init__(self, args, "hermes") + self.list_root = path.dirname(__file__) if self.list_root is None else self.list_root self.collect_excluded_test_lists() self.collect_ignored_test_lists() diff --git a/test/runner/test_js_hermes.py b/test/runner/plugins/hermes/test_js_hermes.py similarity index 100% rename from test/runner/test_js_hermes.py rename to test/runner/plugins/hermes/test_js_hermes.py diff --git a/test/runner/util_hermes.py b/test/runner/plugins/hermes/util_hermes.py similarity index 100% rename from test/runner/util_hermes.py rename to test/runner/plugins/hermes/util_hermes.py diff --git a/test/runner/runner_js_parser.py b/test/runner/plugins/parser/runner_js_parser.py similarity index 76% rename from test/runner/runner_js_parser.py rename to test/runner/plugins/parser/runner_js_parser.py index 2a134867c..7a16b5238 100644 --- a/test/runner/runner_js_parser.py +++ b/test/runner/plugins/parser/runner_js_parser.py @@ -2,16 +2,19 @@ from os import path from runner_base import get_test_id from runner_js import RunnerJS -from test_js_parser import TestJSParser +from plugins.parser.test_js_parser import TestJSParser class RunnerJSParser(RunnerJS): def __init__(self, args): super(RunnerJSParser, self).__init__(args, "parser-js") + self.list_root = path.dirname(__file__) if self.list_root is None else self.list_root self.collect_excluded_test_lists() self.collect_ignored_test_lists() + self.add_directory("parser/js", "js", flags=["--parse-only"]) + def add_directory(self, directory, extension, flags): new_dir = path.join(self.test_root, directory) super(RunnerJSParser, self).add_directory(new_dir, extension, flags) diff --git a/test/runner/test_js_parser.py b/test/runner/plugins/parser/test_js_parser.py similarity index 100% rename from test/runner/test_js_parser.py rename to test/runner/plugins/parser/test_js_parser.py diff --git a/test/runner/runner_js_test262.py b/test/runner/plugins/test262/runner_js_test262.py similarity index 81% rename from test/runner/runner_js_test262.py rename to test/runner/plugins/test262/runner_js_test262.py index 21a880723..ea78f2b1e 100644 --- a/test/runner/runner_js_test262.py +++ b/test/runner/plugins/test262/runner_js_test262.py @@ -2,8 +2,8 @@ from os import path from runner_base import correct_path, get_test_id from runner_js import RunnerJS -from test_js_test262 import TestJSTest262 -from util_test262 import UtilTest262 +from plugins.test262.test_js_test262 import TestJSTest262 +from plugins.test262.util_test262 import UtilTest262 class RunnerJSTest262(RunnerJS): @@ -11,6 +11,7 @@ class RunnerJSTest262(RunnerJS): RunnerJS.__init__(self, args, "test262-ark") self.ignored_name_prefix = "test262" + self.list_root = path.dirname(__file__) if self.list_root is None else self.list_root self.collect_excluded_test_lists(test_name=self.ignored_name_prefix) self.collect_ignored_test_lists(test_name=self.ignored_name_prefix) @@ -21,7 +22,7 @@ class RunnerJSTest262(RunnerJS): self.util = UtilTest262() self.test_root = self.util.generate( self.build_dir, - path.join(self.list_root, "runner/test262harness.js"), + path.join(self.list_root, "test262harness.js"), args.progress) self.test_env.util = self.util diff --git a/test/test262-excluded.txt b/test/runner/plugins/test262/test262-excluded.txt similarity index 100% rename from test/test262-excluded.txt rename to test/runner/plugins/test262/test262-excluded.txt diff --git a/test/test262-flaky-ignored-AOT-FULL.txt b/test/runner/plugins/test262/test262-flaky-ignored-AOT-FULL.txt similarity index 100% rename from test/test262-flaky-ignored-AOT-FULL.txt rename to test/runner/plugins/test262/test262-flaky-ignored-AOT-FULL.txt diff --git a/test/test262-flaky-ignored-AOT.txt b/test/runner/plugins/test262/test262-flaky-ignored-AOT.txt similarity index 100% rename from test/test262-flaky-ignored-AOT.txt rename to test/runner/plugins/test262/test262-flaky-ignored-AOT.txt diff --git a/test/test262-flaky-ignored-INT.txt b/test/runner/plugins/test262/test262-flaky-ignored-INT.txt similarity index 100% rename from test/test262-flaky-ignored-INT.txt rename to test/runner/plugins/test262/test262-flaky-ignored-INT.txt diff --git a/test/test262-flaky-ignored-IRTOC.txt b/test/runner/plugins/test262/test262-flaky-ignored-IRTOC.txt similarity index 100% rename from test/test262-flaky-ignored-IRTOC.txt rename to test/runner/plugins/test262/test262-flaky-ignored-IRTOC.txt diff --git a/test/test262-flaky-ignored-JIT.txt b/test/runner/plugins/test262/test262-flaky-ignored-JIT.txt similarity index 100% rename from test/test262-flaky-ignored-JIT.txt rename to test/runner/plugins/test262/test262-flaky-ignored-JIT.txt diff --git a/test/test262-ignored-AOT-FULL.txt b/test/runner/plugins/test262/test262-ignored-AOT-FULL.txt similarity index 100% rename from test/test262-ignored-AOT-FULL.txt rename to test/runner/plugins/test262/test262-ignored-AOT-FULL.txt diff --git a/test/test262-ignored-AOT.txt b/test/runner/plugins/test262/test262-ignored-AOT.txt similarity index 100% rename from test/test262-ignored-AOT.txt rename to test/runner/plugins/test262/test262-ignored-AOT.txt diff --git a/test/test262-ignored-INT.txt b/test/runner/plugins/test262/test262-ignored-INT.txt similarity index 100% rename from test/test262-ignored-INT.txt rename to test/runner/plugins/test262/test262-ignored-INT.txt diff --git a/test/test262-ignored-IRTOC.txt b/test/runner/plugins/test262/test262-ignored-IRTOC.txt similarity index 100% rename from test/test262-ignored-IRTOC.txt rename to test/runner/plugins/test262/test262-ignored-IRTOC.txt diff --git a/test/test262-ignored-JIT.txt b/test/runner/plugins/test262/test262-ignored-JIT.txt similarity index 98% rename from test/test262-ignored-JIT.txt rename to test/runner/plugins/test262/test262-ignored-JIT.txt index 13504b6cd..fcf97088b 100644 --- a/test/test262-ignored-JIT.txt +++ b/test/runner/plugins/test262/test262-ignored-JIT.txt @@ -44,6 +44,8 @@ language/statements/class/dstr/private-gen-meth-dflt-ary-ptrn-elem-id-iter-val-a language/statements/class/dstr/private-gen-meth-static-ary-ptrn-elem-id-iter-val-array-prototype.js # #9382 language/statements/class/dstr/private-gen-meth-dflt-ary-ptrn-rest-id-direct.js +# #9634 +language/expressions/in/S11.8.7_A1.js # new fails language/white-space/string-horizontal-tab.js diff --git a/test/test262-ignored.txt b/test/runner/plugins/test262/test262-ignored.txt similarity index 100% rename from test/test262-ignored.txt rename to test/runner/plugins/test262/test262-ignored.txt diff --git a/test/test262-long-excluded.txt b/test/runner/plugins/test262/test262-long-excluded.txt similarity index 100% rename from test/test262-long-excluded.txt rename to test/runner/plugins/test262/test262-long-excluded.txt diff --git a/test/runner/test262harness.js b/test/runner/plugins/test262/test262harness.js similarity index 100% rename from test/runner/test262harness.js rename to test/runner/plugins/test262/test262harness.js diff --git a/test/test262skiplist-bco.txt b/test/runner/plugins/test262/test262skiplist-bco.txt similarity index 100% rename from test/test262skiplist-bco.txt rename to test/runner/plugins/test262/test262skiplist-bco.txt diff --git a/test/test262skiplist-flaky-ARM32INT.txt b/test/runner/plugins/test262/test262skiplist-flaky-ARM32INT.txt similarity index 100% rename from test/test262skiplist-flaky-ARM32INT.txt rename to test/runner/plugins/test262/test262skiplist-flaky-ARM32INT.txt diff --git a/test/runner/test_js_test262.py b/test/runner/plugins/test262/test_js_test262.py similarity index 100% rename from test/runner/test_js_test262.py rename to test/runner/plugins/test262/test_js_test262.py diff --git a/test/runner/util_test262.py b/test/runner/plugins/test262/util_test262.py similarity index 100% rename from test/runner/util_test262.py rename to test/runner/plugins/test262/util_test262.py diff --git a/test/runner/plugins_registry.py b/test/runner/plugins_registry.py new file mode 100644 index 000000000..d52886f3f --- /dev/null +++ b/test/runner/plugins_registry.py @@ -0,0 +1,100 @@ +import subprocess +from os import path, listdir, getenv, chdir + +from runner_base import Runner + + +class Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + + +class PluginsRegistry(metaclass=Singleton): + ENV_PLUGIN_PATH = "PLUGIN_PATH" + BUILTIN_PLUGINS = "plugins" + + def __init__(self): + self.registry = {} + self.side_plugins = [] + self.load_from_env() + self.load_builtin_plugins() + + @staticmethod + def filter_builtins(item_list): + return list(filter( + lambda x: not x.startswith("__"), + item_list + )) + + @staticmethod + def my_dir(obj): + return PluginsRegistry.filter_builtins(dir(obj)) + + def load_plugin(self, plugin_name: str, plugin_path: str): + runner_class = list(filter( + lambda x: x.startswith("runner_"), + PluginsRegistry.filter_builtins(listdir(plugin_path)) + )) + runner_class = runner_class.pop() if len(runner_class) > 0 else None + if runner_class is not None: + last_dot = runner_class.rfind(".") + runner_class_name = runner_class[:last_dot] + class_module_name = f"{PluginsRegistry.BUILTIN_PLUGINS}.{plugin_name}.{runner_class_name}" + class_module = __import__(class_module_name) + class_module = getattr(getattr(class_module, plugin_name), runner_class_name) + classes = PluginsRegistry.my_dir(class_module) + classes = list(filter( + lambda x: x.startswith("Runner") and x.lower().endswith(plugin_name.lower()), + classes + )) + class_name = classes.pop() if len(classes) > 0 else None + if class_name is not None: + class_obj = getattr(class_module, class_name) + self.add(plugin_name, class_obj) + + def load_builtin_plugins(self): + starting_path = path.join(path.dirname(__file__), PluginsRegistry.BUILTIN_PLUGINS) + plugin_names = PluginsRegistry.filter_builtins(listdir(starting_path)) + + for plugin_name in plugin_names: + plugin_path = path.join(starting_path, plugin_name) + self.load_plugin(plugin_name, plugin_path) + + def load_from_env(self): + builtin_plugins_path = path.join(path.dirname(__file__), PluginsRegistry.BUILTIN_PLUGINS) + side_plugins = getenv(PluginsRegistry.ENV_PLUGIN_PATH, "").split(path.pathsep) + if len(side_plugins) > 0: + chdir(builtin_plugins_path) + for side_plugin in side_plugins: + if path.exists(side_plugin): + cmd = ["ln", "-s", side_plugin] + subprocess.run(cmd) + self.side_plugins.append(path.join( + path.dirname(__file__), + PluginsRegistry.BUILTIN_PLUGINS, + path.basename(side_plugin) + )) + + def add(self, runner_name: str, runner): + if runner_name not in self.registry: + self.registry[runner_name] = runner + print(f"Registered plugin '{runner_name}' with class '{runner.__name__}'") + else: + raise Exception(f"Plugin '{runner_name}' already registered") + + def get_runner(self, name: str) -> Runner: + return self.registry.get(name, None) + + def is_registered(self, name: str) -> bool: + return name in self.registry.keys() + + def cleanup(self): + if len(self.side_plugins) > 0: + cmd = ["rm"] + for side_plugin in self.side_plugins: + cmd.append(side_plugin) + subprocess.run(cmd) diff --git a/test/runner/runner.py b/test/runner/runner.py index 607e8cb25..975e61669 100644 --- a/test/runner/runner.py +++ b/test/runner/runner.py @@ -1,9 +1,7 @@ import os from dotenv import load_dotenv -from runner_js_hermes import RunnerJSHermes -from runner_js_parser import RunnerJSParser -from runner_js_test262 import RunnerJSTest262 +from plugins_registry import PluginsRegistry from starter import get_args @@ -14,18 +12,22 @@ def main(): args = get_args() - runners = [] - + test_suites = set(args.test_suites[:]) + if args.test262: + test_suites.add("test262") + if args.hermes: + test_suites.add("hermes") if args.regression: - runner_main = RunnerJSParser(args) - runner_main.add_directory("parser/js", "js", flags=["--parse-only"]) - runners.append(runner_main) + test_suites.add("parser") - if args.test262: - runners.append(RunnerJSTest262(args)) + registry = PluginsRegistry() + runners = [] - if args.hermes: - runners.append(RunnerJSHermes(args)) + for test_suite in test_suites: + if registry.is_registered(test_suite): + runner_class = registry.get_runner(test_suite) + # noinspection PyCallingNonCallable + runners.append(runner_class(args)) failed_tests = 0 @@ -33,6 +35,7 @@ def main(): runner.run() failed_tests += runner.summarize() + registry.cleanup() exit(0 if failed_tests == 0 else 1) diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 506e7c46a..1b761277a 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -73,7 +73,7 @@ class Runner: # directory where list files (files with list of ignored, excluded, and other tests) are located # it's either set explicitly to the absolute value or # the current folder (where this python file is located!) parent - self.list_root = args.list_root if args.list_root is not None else current_folder_parent + self.list_root = args.list_root if args.list_root is not None else None # directory where report files for failed tests will be saved - # it's either set explicitly to the absolute value or # the /tmp//reports folder diff --git a/test/runner/starter.py b/test/runner/starter.py index a1f1792f1..51d5e6425 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -31,6 +31,10 @@ def get_args(): parser.add_argument( 'build_dir', type=lambda arg: is_directory(parser, arg), help='build directory') + parser.add_argument( + '--test-suite', action='append', dest='test_suites', + default=[], + help='names of test suites to run') parser.add_argument( '--test262', '-t', action='store_true', dest='test262', default=False, help='run test262 tests') @@ -40,9 +44,6 @@ def get_args(): parser.add_argument( '--hermes', action='store_true', dest='hermes', default=False, help='run Hermes tests') - parser.add_argument( - '--tsc', action='store_true', dest='tsc', - default=False, help='run tsc tests') parser.add_argument( '--test-root', dest='test_root', default=None, type=lambda arg: is_directory(parser, arg), -- Gitee