diff --git a/test/runner/runner_base.py b/test/runner/runner_base.py index 1b761277ac0db501b3eecec5187acde7352ccbdf..d7811d819d71790a6ea8d85ef1bcf25cc9dfd6e8 100644 --- a/test/runner/runner_base.py +++ b/test/runner/runner_base.py @@ -118,7 +118,7 @@ class Runner: # Test chosen to execute can detect itself as excluded one self.excluded_after = 0 - self.update = args.update + self.update_excluded = args.update_excluded def load_tests_from_list(self, list_name): list_path = correct_path(self.list_root, list_name) @@ -163,8 +163,9 @@ class Runner: elif self.explicit_list is not None: test_files.extend(self.load_explicit_tests()) else: - self.load_excluded_tests() - self.load_ignored_tests() + if not self.args.skip_test_lists: + self.load_excluded_tests() + self.load_ignored_tests() if len(test_files) == 0: glob_expression = path.join(directory, f"**/*.{extension}") test_files.extend(fnmatch.filter( @@ -175,7 +176,7 @@ class Runner: self.tests.update(list(map( lambda x: self.create_test(x, flags, is_test_in_list(x, self.ignored_tests)), filter( - lambda x: not is_test_in_list(x, self.excluded_tests), + lambda x: self.update_excluded or not is_test_in_list(x, self.excluded_tests), test_files ) ))) diff --git a/test/runner/runner_js.py b/test/runner/runner_js.py index 6d18d102fcff18ec9e284f03c32b5759447c1da4..79d14a91c09155f6bfb4e3bf4a9b750ceb48517b 100644 --- a/test/runner/runner_js.py +++ b/test/runner/runner_js.py @@ -172,11 +172,14 @@ class RunnerJS(Runner): fail_lists[kind] = [] ignored_still_failed = [] ignored_but_passed = [] + excluded_but_passed = [] + excluded_still_failed = [] self.failed = 0 self.ignored = 0 self.passed = 0 self.excluded_after = 0 + self.excluded = 0 if self.update_excluded else self.excluded timestamp = int(datetime.timestamp(datetime.now())) failed_tests = [] @@ -209,6 +212,9 @@ class RunnerJS(Runner): if test_result.ignored: self.ignored += 1 ignored_still_failed.append(test_result) + elif test_result.path in self.excluded_tests: + excluded_still_failed.append(test_result) + self.excluded += 1 else: self.failed += 1 fail_lists[test_result.fail_kind].append(test_result) @@ -216,15 +222,33 @@ class RunnerJS(Runner): self.passed += 1 if test_result.ignored: ignored_but_passed.append(test_result) + if test_result.path in self.excluded_tests: + excluded_but_passed.append(test_result) total_tests = len(self.tests) + self.excluded if self.args.report_format == ReportFormat.HTML.value: self.create_html_index(failed_tests, total_tests, timestamp) - if not self.update: - for kind in FailKind: - self.summarize_list(kind.name, fail_lists[kind]) + for kind in FailKind: + self.summarize_list(kind.name, fail_lists[kind]) + + if self.update_excluded: + print(f"Update excluded tests:") + print(f"Passed: {len(excluded_but_passed)} - removed from excluded list") + print(f"Still failed: {len(excluded_still_failed)}") + new_excl_list_path = self.excluded_lists[0][:-4] + "_updated.txt" + excluded_still_failed = sorted(map( + lambda x: x.test_id + path.splitext(x.path)[1], + excluded_still_failed + )) + write_2_file(new_excl_list_path, "\n".join(excluded_still_failed)) + passed_excl_list_path = self.excluded_lists[0][:-4] + "_passed.txt" + excluded_but_passed = sorted(map( + lambda x: x.test_id + path.splitext(x.path)[1], + excluded_but_passed + )) + write_2_file(passed_excl_list_path, "\n".join(excluded_but_passed)) print("Summary(%s):" % self.name) print("\033[37mTotal: %5d" % total_tests) diff --git a/test/runner/starter.py b/test/runner/starter.py index 51d5e642548e60aedd7bb7d0a4d88016d42d5314..08deb47a643f12a03376864d30b9083c2779b5e4 100644 --- a/test/runner/starter.py +++ b/test/runner/starter.py @@ -59,11 +59,11 @@ def get_args(): help='format of report files. Possible values html or md. If not set md format is used') parser.add_argument( - '--no-skip', action='store_false', dest='skip', default=True, - help='don\'t use skiplists') + '--skip-test-lists', action='store_true', dest='skip_test_lists', default=False, + help='do not use ignored or excluded lists, run all available tests, report all found failures') parser.add_argument( - '--update', action='store_true', dest='update', default=False, - help='update skiplist') + '--update-excluded', action='store_true', dest='update_excluded', default=False, + help='update list of excluded tests') parser.add_argument( '--update-expected', action='store_true', dest='update_expected', default=False, help='update files with expected results')