From 54e0fd0fd0e578bdd519a69276a21111126fe1e3 Mon Sep 17 00:00:00 2001 From: vagin ivan Date: Wed, 19 Oct 2022 17:13:04 +0300 Subject: [PATCH] Check hermes tests output with FileCheck Signed-off-by: vagin ivan --- test/hermes-excluded-AOT-FULL.txt | 2 +- test/hermes-excluded.txt | 15 ++++++++++++- test/hermes-ignored-AOT-FULL.txt | 2 ++ test/hermes-ignored-AOT.txt | 2 ++ test/hermes-ignored-JIT.txt | 2 ++ test/runner/test_js_hermes.py | 8 +++---- test/runner/util_hermes.py | 36 +++++++++++++++++++------------ 7 files changed, 46 insertions(+), 21 deletions(-) create mode 100644 test/hermes-ignored-AOT-FULL.txt create mode 100644 test/hermes-ignored-AOT.txt create mode 100644 test/hermes-ignored-JIT.txt diff --git a/test/hermes-excluded-AOT-FULL.txt b/test/hermes-excluded-AOT-FULL.txt index b93473f83..960506eb3 100644 --- a/test/hermes-excluded-AOT-FULL.txt +++ b/test/hermes-excluded-AOT-FULL.txt @@ -1 +1 @@ -env.js \ No newline at end of file +env.js diff --git a/test/hermes-excluded.txt b/test/hermes-excluded.txt index 07da820a4..18daf7375 100644 --- a/test/hermes-excluded.txt +++ b/test/hermes-excluded.txt @@ -242,7 +242,6 @@ rgb-to-grey.js set.js setTimeout-flushed-after-throw.js setTimeout.js -setter-stack-overflow.js source-url-error.js spread-arguments.js stacktrace-bound.js @@ -277,3 +276,17 @@ for_in_bigloop.js ordered-hash-map-invariant.js exp.js sampling-profiler.js + +# Frontend test (ReferenceError: doSomething is not definded) +debug-levels.js +# ReferenceError: HermesInternal is not definded +hermes-es6-promise.js +hermes-internal-get-function-location.js +hermes-internal.js +# ReferenceError: setTimeout is not definded +promise-jobs-scheduled-in-scripts.js +promise-jobs-scheduled-in-tasks.js +# ReferenceError: k is not defined "for (k = 0; k < n; k++)" +test-direct-og-explicit-full-gc.js +test-direct-og-full-gc-reached-via-alloc.js +test-direct-og-tti-reached.js diff --git a/test/hermes-ignored-AOT-FULL.txt b/test/hermes-ignored-AOT-FULL.txt new file mode 100644 index 000000000..d1090b8a4 --- /dev/null +++ b/test/hermes-ignored-AOT-FULL.txt @@ -0,0 +1,2 @@ +setter-stack-overflow.js + diff --git a/test/hermes-ignored-AOT.txt b/test/hermes-ignored-AOT.txt new file mode 100644 index 000000000..d1090b8a4 --- /dev/null +++ b/test/hermes-ignored-AOT.txt @@ -0,0 +1,2 @@ +setter-stack-overflow.js + diff --git a/test/hermes-ignored-JIT.txt b/test/hermes-ignored-JIT.txt new file mode 100644 index 000000000..d1090b8a4 --- /dev/null +++ b/test/hermes-ignored-JIT.txt @@ -0,0 +1,2 @@ +setter-stack-overflow.js + diff --git a/test/runner/test_js_hermes.py b/test/runner/test_js_hermes.py index f360ebaec..0e934ef89 100644 --- a/test/runner/test_js_hermes.py +++ b/test/runner/test_js_hermes.py @@ -15,8 +15,6 @@ class TestJSHermes(TestJS): self.util = self.test_env.util def do_run(self): - self.expected = self.util.get_expected_value(self.path) - test_abc = path.join(self.tmp_dir, FOLDER_FOR_INTERMEDIATE_FILES, f"{self.test_id}.abc") test_an = path.join(self.tmp_dir, FOLDER_FOR_INTERMEDIATE_FILES, f"{self.test_id}.an") @@ -60,10 +58,10 @@ class TestJSHermes(TestJS): self.passed, self.report, self.fail_kind = self.run_runtime( test_an, test_abc, - lambda o, e, rc: self.ark_validate_result(o, e, rc, self.expected) + lambda o, e, rc: self.ark_validate_result(o, e, rc) ) return self - def ark_validate_result(self, actual_output, _1, _2, expected_output): - return purify(actual_output) == purify(expected_output) + def ark_validate_result(self, actual_output, _1, return_code): + return self.util.run_filecheck(self.path, actual_output) and return_code == 0 diff --git a/test/runner/util_hermes.py b/test/runner/util_hermes.py index 8d9203b06..a50b0bbd6 100644 --- a/test/runner/util_hermes.py +++ b/test/runner/util_hermes.py @@ -1,5 +1,6 @@ import re from os import getenv +import subprocess import utils @@ -10,7 +11,6 @@ HERMES_REVISION = "HERMES_REVISION" class UtilHermes: def __init__(self, verbose=False): - self.print_expr = re.compile(r"^\s*print\((?P.+)\)", re.MULTILINE) self.check_expr = re.compile(r"^\s*//\s?(?:CHECK-NEXT|CHECK-LABEL|CHECK):(.+)", re.MULTILINE) self.verbose = verbose @@ -33,17 +33,25 @@ class UtilHermes: source_path=source_path ) - def get_expected_value(self, input_file): - with open(input_file, 'r') as fp: + def run_filecheck(self, test_file, actual_output): + with open(test_file, 'r') as fp: input_str = fp.read() - result = "\n".join(map( - lambda x: x.strip(), - self.check_expr.findall(input_str) - )) - if len(result) == 0: - prints = self.print_expr.findall(input_str) - if len(prints) > 0 and self.verbose: - print(f"Test {input_file} contains {len(prints)} calls of print," + - " but does not contain any check." + - " Please correct the test") - return result + if not re.match(self.check_expr, input_str): + return True + + cmd = ['FileCheck-14', test_file] + try: + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + try: + process.stdin.write(actual_output.encode('utf-8')) + process.communicate(timeout=10) + return_code = process.returncode + except subprocess.TimeoutExpired as e: + print(f"{' '.join(cmd)} failed with {e}") + process.kill() + return_code = -1 + except Exception as e: + print(f"{' '.join(cmd)} failed with {e}") + return_code = -1 + + return return_code == 0 -- Gitee