diff --git a/src/build/comment.yaml b/src/build/comment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3eb8038302d6870b70360b676a72fcd85242087b --- /dev/null +++ b/src/build/comment.yaml @@ -0,0 +1,2 @@ +openeuler: + arch_list: ['aarch64','arm'] \ No newline at end of file diff --git a/src/build/gitee_comment_openeuler.py b/src/build/gitee_comment_openeuler.py new file mode 100644 index 0000000000000000000000000000000000000000..866773239d072a51d9b5cf0d789132eaf743e550 --- /dev/null +++ b/src/build/gitee_comment_openeuler.py @@ -0,0 +1,460 @@ +# -*- coding: utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, +# EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, +# MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: +# Create: 2020-09-23 +# Description: comment pr with build result +# ********************************************************************************** +""" +import os +import re +import stat +import sys +import logging.config +import json +import argparse +import warnings +import yaml +from urllib.parse import unquote + +from yaml.error import YAMLError +from src.ac.framework.ac_result import ACResult, SUCCESS +from src.proxy.gitee_proxy import GiteeProxy +from src.proxy.kafka_proxy import KafkaProducerProxy +from src.proxy.jenkins_proxy import JenkinsProxy +from src.utils.dist_dataset import DistDataset + + +class Comment(object): + """ + comments process + """ + + def __init__(self, pr, jenkins_proxy, *check_item_comment_files): + """ + + :param pr: pull request number + """ + self._pr = pr + self._check_item_comment_files = check_item_comment_files + self._up_builds = [] + self._up_up_builds = [] + self._get_upstream_builds(jenkins_proxy) + self.ac_result = {} + self.compare_package_result = {} + self.check_item_result = {} + self._arch_list=self.get_arch_list() + + @staticmethod + def get_arch_list(): + comment_dir = os.path.dirname(os.path.realpath(__file__))+"/comment.yaml" + with open(comment_dir, "r") as f: + try: + arch_list = yaml.safe_load(f) + arch_list = arch_list["openeuler"]["arch_list"] + except YAMLError: + logger.exception("illegal yaml format of check item comment file ") + return arch_list + + def comment_build(self, gitee_proxy): + """ + 构建结果 + :param jenkins_proxy: + :param gitee_proxy: + :return: + """ + comments = self._comment_build_html_format() + gitee_proxy.comment_pr(self._pr, "\n".join(comments)) + + return "\n".join(comments) + + def comment_at(self, committer, gitee_proxy): + """ + 通知committer + @committer + :param committer: + :param gitee_proxy: + :return: + """ + gitee_proxy.comment_pr(self._pr, "@{}".format(committer)) + + def check_build_result(self): + """ + build result check + :return: + """ + build_result = sum([ACResult.get_instance(build["result"]) for build in self._up_builds], SUCCESS) + return build_result + + def _get_upstream_builds(self, jenkins_proxy): + """ + get upstream builds + :param jenkins_proxy: + :return: + """ + base_job_name = os.environ.get("JOB_NAME") + base_build_id = os.environ.get("BUILD_ID") + base_build_id = int(base_build_id) + logger.debug("base_job_name: %s, base_build_id: %s", base_job_name, base_build_id) + base_build = jenkins_proxy.get_build_info(base_job_name, base_build_id) + logger.debug("get base build") + self._up_builds = jenkins_proxy.get_upstream_builds(base_build) + if self._up_builds: + logger.debug("get up_builds") + self._up_up_builds = jenkins_proxy.get_upstream_builds(self._up_builds[0]) + + def _comment_build_html_format(self): + """ + 组装构建信息，并评论pr + :param jenkins_proxy: JenkinsProxy object + :return: + """ + comments = ["", self.comment_html_table_th()] + + if self._up_up_builds: + logger.debug("get up_up_builds") + comments.extend(self._comment_of_ac(self._up_up_builds[0])) + if self._up_builds: + comments.extend(self._comment_of_check_item(self._up_builds)) + + comments.append("
") + return comments + + def _comment_of_ac(self, build): + """ + 组装门禁检查结果 + :param build: Jenkins Build object，门禁检查jenkins构建对象 + :return: + """ + if "ACL" not in os.environ: + logger.debug("no ac check") + return [] + + try: + acl = json.loads(os.environ["ACL"]) + logger.debug("ac result: %s", acl) + except ValueError: + logger.exception("invalid ac result format") + return [] + + comments = [] + + for index, item in enumerate(acl): + ac_result = ACResult.get_instance(item["result"]) + if index == 0: + build_url = build["url"] + comments.append(self.__class__.comment_html_table_tr( + item["name"], ac_result.emoji, ac_result.hint, + "{}{}".format(build_url, "console"), build["number"], rowspan=len(acl))) + else: + comments.append(self.__class__.comment_html_table_tr_rowspan( + item["name"], ac_result.emoji, ac_result.hint)) + self.ac_result[item["name"]] = ac_result.hint + logger.info("ac comment: %s", comments) + + return comments + + def match(self,name, comment_file): + for arch_name in self._arch_list: + if "/"+arch_name+"/" in name and "_"+arch_name+"_" in comment_file: + return True + return False + + def _comment_of_check_item(self, builds): + """ + check item comment + :param builds: + :return: + """ + comment_file_dict = self._get_comment_file_dict() + build_dict = self._get_build_dict(builds) + tr_list = [] + for arch_name,arch_result in build_dict.items(): + arch_name_rows_span = 0 + arch_comment_dict = comment_file_dict.get(arch_name,{}) + is_first = True + first_image = "" + for image_name, image_result in arch_result.items(): + image_rows_span = len(arch_comment_dict.get(image_name,{})) + arch_name_rows_span += 1 + image_rows_span + if is_first: + first_image = image_name + is_first = False + + for image_name, image_result in arch_result.items(): + image_name_rows_span = len(arch_comment_dict.get(image_name,{})) + 1 + if image_name == first_image: + tr_list.append({"arch_name": arch_name, "arch_name_rows_span": arch_name_rows_span, \ + "image_name": image_name, "image_name_rows_span": image_name_rows_span, "check_name": "build", \ + "status": image_result["result"], "build_url": image_result["build_url"], "build_no": image_result["build_no"]}) + for check_name, check_result in arch_comment_dict.get(image_name,{}).items(): + tr_list.append({"check_name": check_name,"status": check_result}) + else: + tr_list.append({"image_name": image_name, "image_name_rows_span": image_name_rows_span, "check_name": "build", \ + "status": image_result["result"], "build_url": image_result["build_url"], "build_no": image_result["build_no"]}) + for check_name,check_result in arch_comment_dict.get(image_name,{}).items(): + tr_list.append({"check_name": check_name, "status": check_result}) + + comments = [] + for tr in tr_list: + tr_str = "" + if "arch_name" in tr: + tr_str += "{}".format(tr.get("arch_name_rows_span"), tr.get("arch_name")) + if "image_name" in tr: + tr_str += "{}".format(tr.get("image_name_rows_span"), tr.get("image_name")) + tr_str += "{}".format(tr.get("check_name")) + ac_result = ACResult.get_instance(tr.get("status")) + tr_str += "{}{}".format(ac_result.emoji, ac_result.hint) + if "build_url" in tr: + tr_str += "#{}".format(\ + tr.get("image_name_rows_span"),\ + "{}{}".format(tr.get("build_url"), "console"),\ + tr.get("build_no")) + tr_str += "" + comments.append(tr_str) + + return comments + + def _get_build_dict(self,builds): + res_dict = {} + for build in builds: + name, _ = JenkinsProxy.get_job_path_build_no_from_build_url(build["url"]) + status = build["result"] + build_url = build["url"] + + is_continue = True + arch = "" + for arch_name in self._arch_list: + if "/"+arch_name+"/" in name: + arch = arch_name + is_continue = False + break + if is_continue: + continue + current_arch = res_dict.get(arch, {}) + image_name = re.compile(r'\((.+)\)').search(unquote(name)).group(1) + current_arch[image_name] = {"result": status, "build_url": build_url, "build_no": build["number"]} + res_dict[arch] = current_arch + return res_dict + + + def _get_comment_file_dict(self): + res_dict = {} + for check_comment_file in self._check_item_comment_files: + current_name = "" + current_arch = {} + for arch in self._arch_list: + if "_" + arch + "_" in check_comment_file: + current_name = arch + current_arch = res_dict.get(arch, {}) + break + if os.path.exists(check_comment_file): + with open(check_comment_file, "r") as f: + image_name = re.compile(r'\((.+)\)').search(check_comment_file).group(1) + image_dict = current_arch.get(image_name, {}) + try: + content = yaml.safe_load(f) + except YAMLError: # yaml base exception + logger.exception("illegal yaml format of check item comment file ") + logger.debug("comment: %s", content) + for item in content: + image_dict[item.get("name")] = item.get("result") + current_arch[image_name] = image_dict + else: + logger.info(check_comment_file+" is not exists") + continue + res_dict[current_name] = current_arch + + return res_dict + + + @classmethod + def comment_html_table_th(cls): + """ + table header + """ + return "Check Name Build Result Build Details" + + @classmethod + def comment_html_table_tr(cls, name, icon, status, href, build_no, hashtag=True, rowspan=1): + """ + one row or span row + """ + return "{} {}{} " \ + "{}{}".format( + name, icon, status, rowspan, href, "#" if hashtag else "", build_no) + + @classmethod + def comment_html_table_tr_rowspan(cls, name, icon, status): + """ + span row + """ + return "{} {}{}".format(name, icon, status) + + def _get_job_url(self, comment_url): + """ + get_job_url + :param url: + :return: + """ + build_urls = {"trigger": self._up_up_builds[0]["url"], + "comment": os.path.join(comment_url, os.environ.get("BUILD_ID")) + } + for build in self._up_builds: + arch = "" + try: + arch_index = 3 + list_step = 2 + if build["url"]: + job_path = re.sub(r"http[s]?://", "", build["url"]) + arch = job_path.split("/")[::list_step][arch_index] + except IndexError: + logger.info("get arch from job failed, index error.") + except KeyError: + logger.info("not find build url key") + if arch: + build_urls[arch] = build["url"] + + return build_urls + + def _get_all_job_result(self, check_details): + """ + get_all_job_result + :return: + """ + + check_details["static_code"] = self.ac_result + for arch, arch_result in self.check_item_result.items(): + if self.compare_package_result.get(arch): + arch_result["compare_package"] = self.compare_package_result.get(arch) + check_details[arch] = arch_result + + return check_details + + def get_all_result_to_kafka(self, comment_url): + """ + 名称 类型 必选 说明 + build_urls 字典 是 包含多个门禁工程链接和显示文本 + check_total 字符串 是 门禁整体结果 + check_details 字典 是 门禁各个检查项结果 + :return: + """ + check_details = {} + build_urls = self._get_job_url(comment_url) + self._get_all_job_result(check_details) + + if self.check_build_result() == SUCCESS: + check_total = 'SUCCESS' + else: + check_total = 'FAILED' + + all_dict = {"build_urls": build_urls, + "check_total": check_total, + "check_details": check_details + } + logger.info("all_dict = %s", all_dict) + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + modes = stat.S_IWUSR | stat.S_IRUSR + try: + with os.fdopen(os.open("build_result.yaml", flags, modes), "w") as f: + yaml.safe_dump(all_dict, f) + except IOError: + logger.exception("save build result file exception") + + +def init_args(): + """ + init args + :return: + """ + parser = argparse.ArgumentParser() + parser.add_argument("-p", type=int, dest="pr", help="pull request number") + parser.add_argument("-m", type=str, dest="comment_id", help="uniq comment id") + parser.add_argument("-c", type=str, dest="committer", help="commiter") + parser.add_argument("-o", type=str, dest="owner", help="gitee owner") + parser.add_argument("-r", type=str, dest="repo", help="repo name") + parser.add_argument("-t", type=str, dest="gitee_token", help="gitee api token") + + parser.add_argument("-b", type=str, dest="jenkins_base_url", default="https://openeulerjenkins.osinfra.cn/", + help="jenkins base url") + parser.add_argument("-u", type=str, dest="jenkins_user", help="repo name") + parser.add_argument("-j", type=str, dest="jenkins_api_token", help="jenkins api token") + parser.add_argument("-f", type=str, dest="check_result_file", default="", help="compare package check item result") + parser.add_argument("-a", type=str, dest="check_item_comment_files", nargs="*", help="check item comment files") + + parser.add_argument("--disable", dest="enable", default=True, action="store_false", help="comment to gitee switch") + + return parser.parse_args() + + +if "__main__" == __name__: + args = init_args() + if not args.enable: + sys.exit(0) + + _ = not os.path.exists("log") and os.mkdir("log") + logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) + logging.config.fileConfig(logger_conf_path) + logger = logging.getLogger("build") + + dd = DistDataset() + dd.set_attr_stime("comment.job.stime") + + # gitee pr tag + gp = GiteeProxy(args.owner, args.repo, args.gitee_token) + gp.delete_tag_of_pr(args.pr, "ci_processing") + + jp = JenkinsProxy(args.jenkins_base_url, args.jenkins_user, args.jenkins_api_token) + url, build_time, reason = jp.get_job_build_info(os.environ.get("JOB_NAME"), int(os.environ.get("BUILD_ID"))) + dd.set_attr_ctime("comment.job.ctime", build_time) + dd.set_attr("comment.job.link", url) + dd.set_attr("comment.trigger.reason", reason) + + dd.set_attr_stime("comment.build.stime") + + comment = Comment(args.pr, jp, *args.check_item_comment_files) \ + if args.check_item_comment_files else Comment(args.pr, jp) + logger.info("comment: build result......") + comment_content = comment.comment_build(gp) + dd.set_attr_etime("comment.build.etime") + dd.set_attr("comment.build.content.html", comment_content) + + if comment.check_build_result() == SUCCESS: + gp.delete_tag_of_pr(args.pr, "ci_failed") + gp.create_tags_of_pr(args.pr, "ci_successful") + dd.set_attr("comment.build.tags", ["ci_successful"]) + dd.set_attr("comment.build.result", "successful") + if args.check_result_file: + comment.comment_compare_package_details(gp, args.check_result_file) + else: + gp.delete_tag_of_pr(args.pr, "ci_successful") + gp.create_tags_of_pr(args.pr, "ci_failed") + dd.set_attr("comment.build.tags", ["ci_failed"]) + dd.set_attr("comment.build.result", "failed") + if args.owner != "openeuler": + comment.get_all_result_to_kafka(url) + + logger.info("comment: at committer......") + comment.comment_at(args.committer, gp) + + dd.set_attr_etime("comment.job.etime") + + # suppress python warning + warnings.filterwarnings("ignore") + logging.getLogger("elasticsearch").setLevel(logging.WARNING) + logging.getLogger("kafka").setLevel(logging.WARNING) + + # upload to es + kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) + query = {"term": {"id": args.comment_id}} + script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict())