diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 0829957e1016f287626a40ba8cc21df6c0af0b3a..c15fef2e4ac8ec8bbbc380435874beeeac8e08a7 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -158,6 +158,23 @@ class ExtraWork(object): except IOError: logger.exception("save check abi comment exception") + def compare_rpm_diff(self, result_path, pr_link, ignore, check_result_file, pr_commit_json_file): + """ + 对比两个版本rpm包之间的差异，根据差异找到受影响的rpm包 + :param result_path: + :param pr_link: + :param ignore: + :param check_result_file: + :param pr_commit_json_file: + :return: + """ + logging.info("compare package start") + compare_package = ComparePackage(logger=logger) + result = compare_package.output_result_to_console(result_path, pr_link, ignore, self._repo, + check_result_file, pr_commit_json_file) + logging.info("compare package result:%s", result) + logging.info("compare package finish") + def check_install_rpm(self, branch_name, arch, install_root): """ 检查生成的rpm是否可以安装 @@ -227,6 +244,18 @@ def checkabi(config, extrawork): extrawork.check_rpm_abi(config.rpm_repo_url, config.arch, config.output, config.committer, config.comment_file, config.obs_addr, config.branch_name, config.obs_repo_url) + +def comparepackage(config, extrawork): + """ + compare two packages + :param config: args + :param extrawork: + :return: + """ + extrawork.compare_rpm_diff(config.json_path, config.pr_link, config.ignore, config.check_result_file, + config.pr_commit_json_file) + + def checkinstall(config, extrawork): """ check install @@ -267,6 +296,17 @@ if "__main__" == __name__: parser_checkabi.add_argument("-e", type=str, dest="comment_file", help="compare package result comment") parser_checkabi.set_defaults(func=checkabi) + # 添加子命令 checkinstall + parser_comparepackage = subparsers.add_parser('comparepackage', help='add help') + parser_comparepackage.add_argument("-f", type=str, dest="check_result_file", + help="compare package check item result") + parser_comparepackage.add_argument("-j", type=str, dest="json_path", help="compare package json path") + parser_comparepackage.add_argument("-i", "--ignore", action="store_true", default=False, help="ignore or not") + parser_comparepackage.add_argument("-pr", type=str, dest="pr_link", help="PR link") + parser_comparepackage.add_argument("-pr_commit", type=str, dest="pr_commit_json_file", help="PR commit file difference") + parser_comparepackage.add_argument("-p", "--package", type=str, help="obs package") + parser_comparepackage.set_defaults(func=comparepackage) + # 添加子命令 checkinstall parser_checkinstall = subparsers.add_parser('checkinstall', help='add help') parser_checkinstall.add_argument("-r", type=str, dest="branch_name", help="obs project name") @@ -287,6 +327,7 @@ if "__main__" == __name__: from src.build.build_rpm_package import BuildRPMPackage from src.build.related_rpm_package import RelatedRpms from src.utils.check_abi import CheckAbi + from src.utils.compare_package import ComparePackage from src.utils.check_conf import CheckConfig ew = ExtraWork(args.package, args.rpmbuild_dir) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 5d778bca4c7cfb32e400170fadd1fb6b3b54e00e..8ffd5e5c965ab85badd4392338a88c59b9bbd3af 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -55,6 +55,18 @@ class Comment(object): return "\n".join(comments) + def comment_compare_package_details(self, gitee_proxy, check_result_file): + """ + compare package结果 + :param jenkins_proxy: + :param gitee_proxy: + :return: + """ + comments = self._comment_of_compare_package_details(check_result_file) + gitee_proxy.comment_pr(self._pr, "\n".join(comments)) + + return "\n".join(comments) + def comment_at(self, committer, gitee_proxy): """ 通知committer @@ -159,6 +171,74 @@ class Comment(object): name, ac_result.emoji, ac_result.hint, "{}{}".format(build_url, "console"), build.buildno)) logger.info("build comment: %s", comments) + return comments + + def _comment_of_compare_package_details(self, check_result_file): + """ + compare package details + :param: + :return: + """ + comments = [" " + ""] + + def match(name, comment_file): + arch = '' + if "aarch64" in name and "aarch64" in comment_file: + arch = "aarch64" + return True, arch + if "x86-64" in name and "x86_64" in comment_file: + arch = "x86_64" + return True, arch + return False, arch + + for result_file in check_result_file.split(","): + logger.info("check_result_file: %s", result_file) + if not os.path.exists(result_file): + logger.info("%s not exists", result_file) + continue + for build in self._up_builds: + name = build.job._data["fullName"] + logger.info("check build %s", name) + arch_result, arch_name = match(name, result_file) + if not arch_result: # 找到匹配的jenkins build + continue + logger.info("build \"%s\" match", name) + + status = build.get_status() + logger.info("build state: %s", status) + if ACResult.get_instance(status) == SUCCESS: # 保证build状态成功 + with open(result_file, "r") as f: + try: + content = yaml.safe_load(f) + except YAMLError: # yaml base exception + logger.exception("illegal yaml format of check abi comment file ") + logger.info("comment: %s", content) + + for index, item in enumerate(content): + rpm_name = content.get(item) + if item == "diff rpms": + continue + if rpm_name: + result = "FAILED" + else: + result = "SUCCESS" + compare_result = ACResult.get_instance(result) + row_len = len(content) - 1 + if index == 0: + comments.append(" " + "" + "".format(row_len, arch_name, item, "
".join(rpm_name), + compare_result.emoji, compare_result.hint, row_len, + "{}{}".format(build.get_build_url(), "console"), "#", + build.buildno)) + else: + comments.append("" + .format(item, "
".join(rpm_name), compare_result.emoji, + compare_result.hint)) + + logger.info("compare package comment: %s", comments) + comments.append("
Arch Name Ckeck Items Rpm Name Ckeck ResultBuild Details
compare_package({}) {} {}{}{} {}{}
{} {} {}{}
") return comments @@ -247,7 +327,7 @@ def init_args(): parser.add_argument("-b", type=str, dest="jenkins_base_url", help="jenkins base url") parser.add_argument("-u", type=str, dest="jenkins_user", help="repo name") parser.add_argument("-j", type=str, dest="jenkins_api_token", help="jenkins api token") - + parser.add_argument("-f", type=str, dest="check_result_file", help="compare package check item result") parser.add_argument("-a", type=str, dest="check_abi_comment_files", nargs="*", help="check abi comment files") parser.add_argument("--disable", dest="enable", default=True, action="store_false", help="comment to gitee switch") @@ -298,6 +378,7 @@ if "__main__" == __name__: gp.create_tags_of_pr(args.pr, "ci_successful") dd.set_attr("comment.build.tags", ["ci_successful"]) dd.set_attr("comment.build.result", "successful") + comment.comment_compare_package_details(gp, args.check_result_file) else: gp.create_tags_of_pr(args.pr, "ci_failed") dd.set_attr("comment.build.tags", ["ci_failed"]) diff --git a/src/utils/compare_package.py b/src/utils/compare_package.py new file mode 100644 index 0000000000000000000000000000000000000000..e5fc42486576a8258935757619e4bde6e7fbf9c2 --- /dev/null +++ b/src/utils/compare_package.py @@ -0,0 +1,264 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-12-01 +# Description: check compare package +# ********************************************************************************** +""" +import datetime +import os +import sys +import json +import logging +import re + +import yaml +import prettytable as pt + + +class ComparePackage(object): + """compare package functions""" + + all_check_item = ["rpm abi", "rpm kabi", "drive kabi", "rpm jabi", "rpm config", + "rpm kconfig", "rpm provides", "rpm requires", "rpm files"] + + def __init__(self, logger=logging): + self.logger = logger + + def get_dict(self, key_list, data): + """ + 获取字典value + :param key_list: key列表 + :param data: 字典 + :return: + """ + if not isinstance(data, dict): + return None + for key, value in data.items(): + if key == key_list[0]: + if len(key_list) > 1: + key_list = key_list[1:] + result = self.get_dict(key_list, value) + return result + else: + return value + + @staticmethod + def rpm_name(rpm): + """ + 返回rpm包名称 + :param rpm: + :return: + """ + m = re.match(r"^(.+)-.+-.+", rpm) + + if m: + return m.group(1) + else: + return rpm + + def show_rpm_diff(self, compare_details): + """ + 输出rpm包差异 + :param compare_details:差异详情 + :return: + """ + tb = pt.PrettyTable(hrules=True) + tb.field_names = ["新增rpm", "删除rpm", "变更rpm"] + diff_rpm = [] + + for key in ["more", "less", "diff"]: + key_list = [key, "%s_details" % key] + details = self.get_dict(key_list, compare_details) + if details: + diff_rpm.append("\n".join(details)) + else: + diff_rpm.append("") + tb.add_row(diff_rpm) + print(tb) + tb.clear() + + def get_check_item_dict(self, all_item_dict, diff_details): + """ + 获取检查项详情字典 + :param all_item_dict: + :param diff_details: + :return: + """ + rpm_name_list = diff_details.keys() + + for check_item in self.all_check_item: + item_dict = all_item_dict.get(check_item) if all_item_dict.get(check_item) else {} + for rpm_name in rpm_name_list: + rpm_dict = {} + result = self.get_dict([rpm_name, check_item], diff_details) + if result: + rpm_dict[rpm_name] = result + if rpm_dict: + item_dict.update(rpm_dict) + if item_dict: + all_item_dict[check_item] = item_dict + + def show_diff_details(self, diff_details): + """ + 显示有diff差异的rpm包的所有差异详情 + :param diff_details: + :return: + """ + all_item_dict = {} + self.get_check_item_dict(all_item_dict, diff_details) + + for item in self.all_check_item: + item_result = all_item_dict.get(item) + if not item_result: + continue + tb = pt.PrettyTable(hrules=True) + tb.field_names = ["%s变更列表" % item, "新增", "删除", "变更"] + rpm_list = item_result.keys() + for rpm in rpm_list: + rpm_details = item_result.get(rpm) + more_value = less_value = diff_values = "" + + for key, value in rpm_details.items(): + if key == "more": + more_value = "\n".join(value) + elif key == "less": + less_value = "\n".join(value) + elif key == "diff": + diff_value = value.get("old") + diff_values = "\n".join(diff_value) + tb.add_row([rpm, more_value, less_value, diff_values]) + + print(tb) + tb.clear() + + def output_result_to_console(self, json_file, pr_link, ignore, repo, check_result_file, pr_commit_json_file): + """ + 解析结果文件并输出展示到jenkins上 + :param json_file: 结果文件json + :param pr_link: + :param ignore: + :param repo: + :param check_result_file: + :param pr_commit_json_file: + :return: + """ + all_data = {} + result_dict = {"add rpms": [], "delete rpms": [], "diff rpms": []} + if ignore: + return "SUCCESS" + + if not os.path.exists(json_file): + self.logger.error("%s not exists", json_file) + return "FAILED" + + with open(json_file, "r") as data: + all_data = json.load(data) + # 写入pr link到json文件 + with open(json_file, "w") as data: + all_data["pr_link"] = pr_link + all_data["pr_changelog"] = self.get_pr_changelog(pr_commit_json_file) + json.dump(all_data, data) + compare_result = all_data.get("compare_result") + compare_details = all_data.get("compare_details") + self.logger.info("compare <%s> package %s\n" % (repo, compare_result)) + if not compare_details: + return "FAILED" + # 生成各检查项的结果，输出到check_result_file文件中，后面comment时会用到 + self.result_to_table(compare_details, result_dict) + try: + with open(check_result_file, "w") as f: + yaml.safe_dump(result_dict, f) # list + except IOError: + self.logger.exception("save compare package comment exception") + + # 显示rpm包的变更 + self.show_rpm_diff(compare_details) + # 显示有变更的rpm包的具体差异详情 + diff_details = self.get_dict(["diff", "diff_details"], compare_details) + if diff_details: + self.show_diff_details(diff_details) + sys.stdout.flush() + + if compare_result == "pass": + return "SUCCESS" + else: + return "FAILED" + + @staticmethod + def get_check_item_result(details): + """ + 获取compare package比较结果各子项的详细信息 + :param details: + :return: + """ + rpm_dict = {} + for key, value in details.items(): + for key2, value2 in value.items(): + if key2 == "name": + continue + rpm_list = rpm_dict.get(key2) if rpm_dict.get(key2) else [] + if value2: + rpm_list.append(key) + rpm_dict[key2] = rpm_list + return rpm_dict + + def get_pr_changelog(self, pr_commit_json_file): + """ + 获取更新代码的changelog内容，应承载接口变更检查原因及影响 + :param pr_commit_json_file: gitee PR提交对应的变更文件信息, json格式 + :return: 返回spec文件中新增的changelog内容 + """ + if not os.path.exists(pr_commit_json_file): + self.logger.error("%s not exists", pr_commit_json_file) + all_data = {} + result = "" + with open(pr_commit_json_file, "r") as data: + all_data = json.load(data) + for item in all_data: + if ".spec" in item["filename"]: + diff = item["patch"]["diff"] + loc = diff.find("%changelog") + if loc == -1: + continue + diff = diff[loc:len(diff)] + list = diff.split('\n') + for str in list: + if len(str) > 0 and str[0] == '+': + result = result + str[1:len(str)] + "\n" + return result + + def result_to_table(self, compare_details, result_dict): + """ + 获取compare package比较结果的详细信息 + :param compare_details: + :param result_dict: + :return: + """ + for compare_item in ["more", "less", "diff"]: + key_list = [compare_item, "%s_details" % compare_item] + details = self.get_dict(key_list, compare_details) + if not details: + continue + if compare_item == "diff": + rpm_dict = self.get_check_item_result(details) + result_dict.update(rpm_dict) + result_dict["diff rpms"] = list(rpm_dict) + else: + rpm_list = [] + for rpm in details: + rpm_list.append(self.rpm_name(rpm)) + if compare_item == "more": + result_dict["add rpms"] = rpm_list + elif compare_item == "less": + result_dict["delete rpms"] = rpm_list