diff --git a/test/maple_test/configs.py b/test/maple_test/configs.py index cf05a2ddfdc96c447cd29ba2760cd5c831d5235f..614f434c249846730a8b756b2fafe549206a77fe 100644 --- a/test/maple_test/configs.py +++ b/test/maple_test/configs.py @@ -67,6 +67,12 @@ def parse_args(): type=complete_path, help="Store test result as xunit xml format at ", ) + test_framework_parser.add_argument( + "--json_output", + metavar="", + type=complete_path, + help="Store test result as josn format at ", + ) test_framework_parser.add_argument( "--debug", action="store_true", help="keep test temp file" ) @@ -182,6 +188,7 @@ def parse_args(): "retry": args.retry, "output": args.output, "xml_output": args.xml_output, + "json_output": args.json_output, "debug": args.debug, "fail_exit": args.fail_exit, "print_type": args.print_type, diff --git a/test/maple_test/main.py b/test/maple_test/main.py index 3fa539a4d385823785e363db2abd2d4ebe8b4a1e..2add556df4819c9b2306ab29ef88c002074aff9c 100644 --- a/test/maple_test/main.py +++ b/test/maple_test/main.py @@ -39,6 +39,7 @@ def main(): cli_running_config = test_suite_config.get("cli_running_config") root = ElementTree.Element("testsuites") + json_result = [] retry = configs.get_val("retry") result = "" @@ -67,40 +68,8 @@ def main(): test_result = task.gen_summary([]) failed |= test_failed result += test_result - - suite = ElementTree.SubElement( - root, - "testsuite", - name="{} {}".format(task.name, test), - tests=str(sum(task.result.values())), - failures=str(task.result[FAIL]), - ) - - for task_set in task.task_set: - for case in sorted(task.task_set[task_set], key=lambda x: x.case_path): - case_node = ElementTree.SubElement( - suite, - "testcase", - name=str(case.case_path), - classname="{}.{}".format(task.cfg_path.parent.name, task_set), - ) - if case.result[0] == UNRESOLVED: - skipped = ElementTree.SubElement(case_node, "skipped") - skipped.text = "No valid command statement was found." - elif case.result[0] == FAIL: - failure = ElementTree.SubElement(case_node, "failure") - failure.text = "Path: {}\n".format(case.case_path) - failure.text += "Log Path: {}{}{}.log\n".format( - log_dir, OS_SEP, case.name - ) - failure.text += "Work Dir: {}\n".format(case.work_dir) - failure.text += "Execute commands:\n" - for cmd in case.commands: - failure.text += "EXEC: " + cmd + "\n" - failure.text += "-----" + "\n" - failure.text += "Failed command: " + case.result[-1][1] + "\n" - failure.text += "Return Code: " + str(case.result[-1][0]) + "\n" - failure.text += "Stderr: \n" + case.result[-1][-1] + task.gen_xml_result(root) + json_result.append(task.gen_json_result()) else: logger.info("Test path: {} does not exist, please check".format(test)) @@ -109,6 +78,13 @@ def main(): with xml_output.open("w") as f: f.write(ElementTree.tostring(root).decode("utf-8")) + json_output = configs.get_val("json_output") + if json_output: + with json_output.open("w") as f: + import json + + json.dump(json_result, f, indent=2) + output = configs.get_val("output") if output: if output.exists() and output.is_file(): diff --git a/test/maple_test/run.py b/test/maple_test/run.py index 0db0828466deb69f15fb06a455e1f9afb330c7bb..321727364ed95608691d1e0a276c44823260f45c 100644 --- a/test/maple_test/run.py +++ b/test/maple_test/run.py @@ -133,6 +133,7 @@ def run_commands( else: run_command = run_command_linux + commands_result = [] for command in commands: start = timeit.default_timer() @@ -144,12 +145,19 @@ def run_commands( logger.debug( "Run time: {:.2}, remain time: {:.2}".format(run_time, remain_time) ) + command_result = {} + command_result["cmd"] = command + command_result["return_code"] = return_code + command_result["stdout"] = com_out + command_result["stderr"] = com_err + commands_result.append(command_result) if return_code != 0: - result = (FAIL, (return_code, command, com_err)) + result = (FAIL, commands_result) err = "Failed, Log file at: {}.log".format(log_config[0].get("dir") / name) logger.error(err) break - + else: + result = (PASS, commands_result) if result[0] == PASS: logger.debug("Task executed successfully") handlers = logger.handlers[:] diff --git a/test/maple_test/task.py b/test/maple_test/task.py index f350a20b43c0d8096ee147e23aca68ad63177303..7cc2cbda6efc387f1aae49f4295c6a3c53bc7109 100644 --- a/test/maple_test/task.py +++ b/test/maple_test/task.py @@ -102,10 +102,6 @@ class TestSuiteTask: def __init__(self, test_path, cfg_path, running_config, cli_running_config=None): if cli_running_config is None: cli_running_config = {} - user_test_list = cli_running_config.get("test_list") - user_config_set = cli_running_config.get("user_config_set") - user_config = cli_running_config.get("user_config") - user_env = cli_running_config.get("user_env") self.path = complete_path(test_path) self.cfg_path = cfg_path @@ -374,6 +370,50 @@ class TestSuiteTask: summary += "-" * 120 return summary + def gen_result(self): + from maple_test.test import Result + + results = [] + for task_name in self.task_set: + for task in self.task_set[task_name]: + result = Result( + task.case_path, + self.cfg_path.parent.name, + task_name, + task.result[0], + task.commands, + task.result[-1], + ) + results.append(result) + return results + + def gen_xml_result(self, root): + from xml.etree import ElementTree + + suite = ElementTree.SubElement( + root, + "testsuite", + failures=str(self.result[FAIL]), + tests=str(sum(self.result.values())), + name="{} {}".format(self.name, self.path), + ) + for result in sorted(self.gen_result()): + result.gen_xml(suite) + return suite + + def gen_json_result(self): + json_result = OrderedDict() + json_result["name"] = "{} {}".format(self.name, self.path) + json_result["total"] = sum(self.result.values()) + for status in self.result: + if status == NOT_RUN: + continue + json_result[status] = self.result[status] + json_result["tests"] = [] + for result in sorted(self.gen_result()): + json_result["tests"].append(result.gen_json_result()) + return json_result + class SingleTask: def __init__(self, case, config, running_config): diff --git a/test/maple_test/test.py b/test/maple_test/test.py index fd853ec8620c3e32d27150d6dee0797f1d50c242..9173a66417d9e527b526229bb252aede0006aa24 100644 --- a/test/maple_test/test.py +++ b/test/maple_test/test.py @@ -15,10 +15,12 @@ # See the Mulan PSL v1 for more details. # import shlex +from functools import total_ordering from maple_test.utils import PASS, EXEC_FLAG, EXPECT_FLAG, DEPENDENCE_FLAG from maple_test.utils import read_file from maple_test.utils import split_comment, filter_line +from maple_test.utils import FAIL, UNRESOLVED class Case: @@ -118,3 +120,56 @@ def read_list(path): if not case_list: case_list = {"*"} return case_list, exclude_case_list + + +@total_ordering +class Result: + def __init__(self, case, task, cfg, status, commands, commands_result): + self.case = case + self.task = task + self.cfg = cfg + self.time = None + self.status = status + self.commands = commands + self.commands_result = commands_result + + def gen_xml(self, root): + from xml.etree import ElementTree + + case = ElementTree.SubElement( + root, + "testcase", + name=str(self.case), + classname="{}.{}".format(self.task, self.cfg), + ) + + if self.status == FAIL: + failure = ElementTree.SubElement(case, "failure") + failure.text = "List of commands:\n" + for cmd in self.commands: + failure.text += "EXEC: {}\n".format(cmd) + failure.text += "----\n" + failure.text += self.command_result_to_text(self.commands_result[-1]) + elif self.status == UNRESOLVED: + skipped = ElementTree.SubElement(case, "skipped") + skipped.text = "No valid command statement was found." + + def command_result_to_text(self, result): + text = "EXEC: {}\n".format(result.get("cmd")) + text += "Return code: {}\n".format(result.get("return_code")) + text += "Stdout: \n{}\n".format(result.get("stdout")) + text += "Stderr: \n{}\n".format(result.get("stderr")) + return text + + def gen_json_result(self): + from collections import OrderedDict + + result = OrderedDict() + result["name"] = "{}/{} :: {}".format(self.task, self.case, self.cfg) + result["result"] = self.status + result["commands"] = self.commands + result["output"] = self.commands_result + return result + + def __lt__(self, other): + return self.case < other.case