diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 5cabb6fe194b0abda1bc08f8bbf7676d23a1c5ba..85fce10e877ecafcf3f4cdd11dc755ee0e78ba10 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -42,3 +42,28 @@ openeuler: module: openlibing.check_code entry: CheckCode allow_list: ["pkgship", "kunpengsecl", "release-tools"] +zhengyaohui: + spec: + hint: check_spec_file + module: spec.check_spec + entry: CheckSpec + ignored: ["homepage"] + code: + hint: check_code_style + module: code.check_code_style + entry: CheckCodeStyle + #exclude: True + ignored: ["patch"] + package_yaml: + hint: check_package_yaml_file + module: package_yaml.check_yaml + entry: CheckPackageYaml + ignored: ["fields"] + package_license: + hint: check_package_license + module: package_license.check_license + entry: CheckLicense + sca: + exclude: True + openlibing: + exclude: True diff --git a/src/build/extra_work.py b/src/build/extra_work.py index fbefcc84334d537c19a1ccd4c509bfe73621a02f..440353414ff93c07541a6e4f9670a403fb90eb3c 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -21,9 +21,12 @@ import argparse import logging.config import logging import yaml +import json from src.build.obs_repo_source import OBSRepoSource +SUCCESS = "SUCCESS" +FAILED = "FAILED" class ExtraWork(object): """ @@ -157,6 +160,164 @@ class ExtraWork(object): yaml.safe_dump(comments, f) # list except IOError: logger.exception("save check abi comment exception") + + def get_dict(self, key_list, data): + """ + 获取字典value + :param key_list: key列表 + :param data: 字典 + :return: + """ + if not isinstance(data, dict): + return None + for key, value in data.items(): + if key == key_list[0]: + if len(key_list) > 1: + key_list = key_list[1:] + result = self.get_dict(key_list, value) + return result + else: + return value + + @staticmethod + def get_item_data(item, data, fp): + """ + 输出diff的详情至文件fp中 + :param item: + :param data: + :param fp: + :return: + """ + str_list = [] + if isinstance(data, list): + str_list.append(" {}:\n".format(item)) + for item_data in data: + str_list.append(" {}\n".format(item_data)) + elif isinstance(data, dict): + str_list.append(" {}:\n".format(item)) + for key, value in data.items(): + str_list.append(" {}:\n".format(key)) + if isinstance(value, list): + for i in value: + str_list.append(" {}\n".format(i)) + else: + str_list.append("{}\n".format(value)) + fp.writelines(str_list) + + def show_rpm_diff(self, compare_details, fp): + """ + 输出rpm包差异 + :param compare_details:差异详情 + :param fp: 存放展示结果的文件句柄 + :return: + """ + for key in ["more", "less", "diff"]: + key_list = [key, "%s_details" % key] + details = self.get_dict(key_list, compare_details) + if not details: + continue + if key == "diff": + fp.write("diff[old_rpm_name,new_rpm_name]:\n") + for rpm in details: + new_rpm_name = self.get_dict([rpm, "name", "new"], details) + fp.write("%s,%s\n" %(rpm, new_rpm_name)) + else: + fp.write("%s:\n" % key) + for rpm in details: + fp.write("%s\n" % rpm) + + def output_result_to_console(self, json_file, pr_link, fp): + """ + 解析结果文件并输出展示到jenkins上 + :param json_file: 结果文件json + :param pr_link: pr_link + :param fp: 存放展示结果的文件句柄 + :return: 比对结果 + """ + all_data = {} + diff_item = ["more", "less", "diff"] + + if not os.path.exists(json_file): + logger.error("%s not exists", json_file) + return FAILED + with open(json_file, "r") as data: + all_data = json.load(data) + # 写入pr link到json文件 + with open(json_file, "w") as data: + all_data["pr_link"] = pr_link + json.dump(all_data, data) + compare_result = all_data.get("compare_result") + compare_details = all_data.get("compare_details") + fp.write("compare <%s> package %s\n" %(self._repo, compare_result)) + if not compare_details: + return FAILED + # 显示所有的差异 + self.show_rpm_diff(compare_details, fp) + + # 显示diff的详细差异 + diff_details = self.get_dict(["diff", "diff_details"], compare_details) + for key, value in diff_details.items(): + new_rpm_name = self.get_dict(["name", "new"], value) + fp.write("compare %s,%s not pass\n" % (key, new_rpm_name)) + compare_list = list(value.keys()) + for compare_item in compare_list: + if compare_item == "name": + continue + fp.write("%s:\n" % compare_item) + item_list = [[compare_item, i] for i in diff_item] + for items in item_list: + items_data = self.get_dict(items, value) + if items_data: + self.get_item_data(items[1], items_data, fp) + + if compare_result == "pass": + return SUCCESS + else: + return FAILED + + def compare_rpm_diff(self, package_url, package_arch, output, committer, comment_file, + result_path, pr_link): + """ + 对比两个版本rpm包之间的差异,根据差异找到受影响的rpm包 + :param package_arch: + :param obs_repo_url: + :return: + """ + #get rpms + logging.info("compare package start") + tmp_file = "./tmp_result.txt" + with open(tmp_file, "w") as fp: + result = self.output_result_to_console(result_path, pr_link, fp) + with open(tmp_file, "r") as fp: + logger.info(fp.read()) + + comment = {"name": "compare_package/{}/{}".format(package_arch, self._repo), "result": result, + "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, + os.path.basename(output), package_url)} + + logger.debug("compare package comment: {}".format(comment)) + try: + with open(comment_file, "r") as f: # one repo with multi build package + comments = yaml.safe_load(f) + except IOError as e: + logger.debug("no history compare package comment") + + comments = [] + if os.path.exists(comment_file): + try: + with open(comment_file, "r") as f: # one repo with multi build package + comments = yaml.safe_load(f) + except IOError: + logger.exception("yaml load compare package comment file exception") + + comments.append(comment) + logger.debug("compare package comments: {}".format(comments)) + logging.info("compare package finish") + try: + with open(comment_file, "w") as f: + yaml.safe_dump(comments, f) # list + except IOError: + logger.exception("save compare package comment exception") def check_install_rpm(self, branch_name, arch, install_root): """ @@ -204,34 +365,74 @@ class ExtraWork(object): else: logger.info("install rpm success") +def notify(args, ew): + # run after copy rpm to rpm repo + if ew.is_pkgship_need_notify(args.pkgship_meta): + ew.pkgship_notify( + args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) -if "__main__" == __name__: - args = argparse.ArgumentParser() +def checkabi(args, ew): + # run before copy rpm to rpm repo + ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + args.obs_addr, args.branch_name, args.obs_repo_url) - args.add_argument("-f", type=str, dest="func", choices=("notify", "checkabi", "checkinstall"), help="function") +def comparepackage(args, ew): + ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + args.json_path, args.pr_link) - args.add_argument("-p", type=str, dest="package", help="obs package") - args.add_argument("-a", type=str, dest="arch", help="build arch") - args.add_argument("-c", type=str, dest="committer", help="committer") +def checkinstall(args, ew): + ew.check_install_rpm(args.branch_name, args.arch, args.install_root) - args.add_argument("-d", type=str, dest="rpmbuild_dir", +if "__main__" == __name__: + parser = argparse.ArgumentParser() + parser.add_argument("-p", "--package", type=str, default="src-openeuler", help="obs package") + parser.add_argument("-d", "--rpmbuild_dir", type=str, default="/home/jenkins/agent/buildroot/home/abuild/rpmbuild", help="rpmbuild dir") - - args.add_argument("-n", type=str, dest="notify_url", help="target branch that merged to ") - args.add_argument("-t", type=str, dest="token", default=os.getcwd(), help="obs workspace dir path") - args.add_argument("-u", type=str, dest="notify_user", default="trigger", help="notify trigger user") - args.add_argument("-w", type=str, dest="notify_password", help="notify trigger password") - args.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") - args.add_argument("-m", type=str, dest="pkgship_meta", help="meta from pkgship spec") - - args.add_argument("-o", type=str, dest="output", help="checkabi result") - args.add_argument("-e", type=str, dest="comment_file", help="checkabi result comment") - args.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") - args.add_argument("-s", type=str, dest="obs_addr", help="obs address") - args.add_argument("-r", type=str, dest="branch_name", help="obs project name") - - args.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") - args = args.parse_args() + subparsers = parser.add_subparsers(help='sub-command help') + + #添加子命令 notify + parser_notify = subparsers.add_parser('notify', help='add help') + parser_notify.add_argument("-n", type=str, dest="notify_url", help="target branch that merged to ") + parser_notify.add_argument("-t", type=str, dest="token", default=os.getcwd(), help="obs workspace dir path") + parser_notify.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") + parser_notify.add_argument("-a", type=str, dest="arch", help="build arch") + parser_notify.add_argument("-u", type=str, dest="notify_user", default="trigger", help="notify trigger user") + parser_notify.add_argument("-w", type=str, dest="notify_password", help="notify trigger password") + parser_notify.set_defaults(func=notify) + + #添加子命令 checkabi + parser_checkabi = subparsers.add_parser('checkabi', help='add help') + parser_checkabi.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") + parser_checkabi.add_argument("-a", type=str, dest="arch", help="build arch") + parser_checkabi.add_argument("-o", type=str, dest="output", help="checkabi result") + parser_checkabi.add_argument("-c", type=str, dest="committer", help="committer") + parser_checkabi.add_argument("-s", type=str, dest="obs_addr", help="obs address") + parser_checkabi.add_argument("-r", type=str, dest="branch_name", help="obs project name") + parser_checkabi.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") + parser_checkabi.add_argument("-p", "--package", type=str, help="obs package") + parser_checkabi.add_argument("-e", type=str, dest="comment_file", help="compare package result comment") + parser_checkabi.set_defaults(func=checkabi) + + #添加子命令 comparepackage + parser_comparepackage = subparsers.add_parser('comparepackage', help='add help') + parser_comparepackage.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") + parser_comparepackage.add_argument("-a", type=str, dest="arch", help="build arch") + parser_comparepackage.add_argument("-o", type=str, dest="output", help="checkabi result") + parser_comparepackage.add_argument("-c", type=str, dest="committer", help="committer") + parser_comparepackage.add_argument("-e", type=str, dest="comment_file", help="compare package result comment") + parser_comparepackage.add_argument("-j", type=str, dest="json_path", help="compare package json path") + parser_comparepackage.add_argument("-pr", type=str, dest="pr_link", help="PR link") + parser_comparepackage.add_argument("-p", "--package", type=str, help="obs package") + parser_comparepackage.set_defaults(func=comparepackage) + + #添加子命令 checkinstall + parser_checkinstall = subparsers.add_parser('checkinstall', help='add help') + parser_checkinstall.add_argument("-r", type=str, dest="branch_name", help="obs project name") + parser_checkinstall.add_argument("-a", type=str, dest="arch", help="build arch") + parser_checkinstall.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") + parser_checkinstall.set_defaults(func=checkinstall) + + args = parser.parse_args() _ = not os.path.exists("log") and os.mkdir("log") logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) @@ -243,16 +444,6 @@ if "__main__" == __name__: from src.build.related_rpm_package import RelatedRpms from src.utils.check_abi import CheckAbi from src.utils.check_conf import CheckConfig - + ew = ExtraWork(args.package, args.rpmbuild_dir) - if args.func == "notify": - # run after copy rpm to rpm repo - if ew.is_pkgship_need_notify(args.pkgship_meta): - ew.pkgship_notify( - args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) - elif args.func == "checkabi": - # run before copy rpm to rpm repo - ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - args.obs_addr, args.branch_name, args.obs_repo_url) - elif args.func == "checkinstall": - ew.check_install_rpm(args.branch_name, args.arch, args.install_root) + args.func(args, ew) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 442ebbbe4250f592dca24e1d713be61594d52293..5d7e412b3acd01377e6129a9abf59036b2530fbf 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -103,7 +103,8 @@ class Comment(object): comments.extend(self._comment_of_ac(self._up_up_builds[0])) if self._up_builds: comments.extend(self._comment_of_build(self._up_builds)) - comments.extend(self._comment_of_check_abi(self._up_builds)) + #comments.extend(self._comment_of_check_abi(self._up_builds)) + comments.extend(self._comment_of_compare_package(self._up_builds)) comments.append("") return comments @@ -162,6 +163,51 @@ class Comment(object): return comments + def _comment_of_compare_package(self, builds): + """ + compare package comment + :param builds: + :return: + """ + comments = [] + + def match(name, comment_file): + if "aarch64" in name and "aarch64" in comment_file: + return True + if "x86-64" in name and "x86_64" in comment_file: + return True + return False + + for check_abi_comment_file in self._check_abi_comment_files: + logger.debug("compare package comment file: {}".format(check_abi_comment_file)) + if not os.path.exists(check_abi_comment_file): # check abi评论文件存在 + continue + for build in builds: + name = build.job._data["fullName"] + logger.debug("check build {}".format(name)) + if not match(name, check_abi_comment_file): # 找到匹配的jenkins build + continue + logger.debug("build \"{}\" match".format(name)) + + status = build.get_status() + logger.debug("build state: {}".format(status)) + if ACResult.get_instance(status) == SUCCESS: # 保证build状态成功 + with open(check_abi_comment_file, "r") as f: + try: + content = yaml.safe_load(f) + except YAMLError: # yaml base exception + logger.exception("illegal yaml format of check abi comment file ") + logger.debug("comment: {}".format(content)) + for item in content: + ac_result = ACResult.get_instance(item.get("result")) + comments.append(self.__class__.comment_html_table_tr( + item.get("name"), ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) + break + + logger.info("compare package comment: {}".format(comments)) + + return comments + def _comment_of_check_abi(self, builds): """ check abi comment