diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 4d2c3231541b0f19c3cdb3fa2577258dc56c5e11..718dedb9da7a618d69906f2fd559f17ee10e85c5 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -40,9 +40,7 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl", - "release-tools", "yocto-meta-openeuler", "yocto-embedded-tools", "yocto-poky", "iSulad", - "gcc", "gcc-anti-sca", "A-Ops"] + deny_list: [] openlibing: hint: code module: openlibing.check_code diff --git a/src/ci.log b/src/ci.log new file mode 100644 index 0000000000000000000000000000000000000000..1146610c6961446a2fc4c4b0cffb4ca37a49e49a --- /dev/null +++ b/src/ci.log @@ -0,0 +1,5 @@ +aaa1 +bbb1 +2022-03-22 19:38:38,575 [WARNING] : aaa1 +%(asctime)s %(filename)20s[line:%(lineno)3d] %(levelname)7s : %(message)s +2022-03-22 19:47:08,476 logger.py[line: 38] WARNING : aaa1 diff --git a/src/logger.py b/src/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..675ae8ea376bbe9e6f265e3f434c811eaad2bbf1 --- /dev/null +++ b/src/logger.py @@ -0,0 +1,38 @@ +import os +import logging +import sys + +from src.utils.color_log import CusColoredFormatter + + +def singleton(cls): + instances = {} + + def _singleton(*args, **kwargs): + if cls not in instances: + instances[cls] = cls(*args, **kwargs) + return instances[cls] + + return _singleton + + +@singleton +class Logger: + def __init__(self): + self.logger = logging.getLogger() + + self.sh = logging.StreamHandler(sys.stdout) + self.sh.setFormatter(CusColoredFormatter(fmt="%(log_color)s%(asctime)s [%(levelname)7s] : %(message)s")) + self.sh.setLevel(logging.INFO) + self.fh = logging.FileHandler("{}/ci.log".format(os.path.dirname(__file__))) + self.fh.setFormatter(logging.Formatter( + "%(asctime)s %(filename)20s[line:%(lineno)3d] %(levelname)7s : %(message)s")) + self.fh.setLevel(logging.DEBUG) + self.logger.addHandler(self.sh) + self.logger.addHandler(self.fh) + + +if __name__ == "__main__": + lg = Logger() + lg.logger.warning("test 1") + lg.logger.debug("test 2") \ No newline at end of file diff --git a/src/utils/ci_mistake.py b/src/utils/ci_mistake.py new file mode 100644 index 0000000000000000000000000000000000000000..be06d9c495d43d810b95051e44d3b16e110affa1 --- /dev/null +++ b/src/utils/ci_mistake.py @@ -0,0 +1,240 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-3-21 +# Description: ci mistake report to database by kafka +# ********************************************************************************** +""" +import os +import json +import logging.config +import logging +import re +import datetime +import time +import argparse + +from src.proxy.gitee_proxy import GiteeProxy +from src.proxy.kafka_proxy import KafkaProducerProxy +from src.logger import Logger + + +class CiMistake(object): + """ci mistake functions""" + + support_check_stage = ["check_binary_file", "check_package_license", "check_package_yaml_file", + "check_spec_file", "check_build", "check_install", "compare_package", + "build_exception"] + + support_mistake_type = ["ci", "obs", "infra"] + + def __init__(self, pr_url, gitee_token, committer, commit_at, comment_id): + """ + initial + :param logger: log object + :param pr_url: pr link + :param gitee_token: gitee comment token + :param committer: ci mistake comment committer + :param commit_at: ci mistake comment date + :param comment_id: gitee comment id + :return: + """ + self.logger = Logger() + self.pr_url = pr_url + self.gitee_token = gitee_token + self.committer = committer + self.comment_id = comment_id + ctime = datetime.datetime.strptime(commit_at, "%Y-%m-%dT%H:%M:%S%z") + commit_at_timestamp = time.mktime(ctime.timetuple()) + self.commit_at = commit_at_timestamp + self.owner, self.repo, self.pr_id = CiMistake.get_owner_repo(pr_url) + + def load_build_no_list(self, filepath): + """ + load build number list + :param filepath: + :return: a number list + """ + if not os.path.exists(filepath): + self.logger.error("%s not exists", os.path.basename(filepath)) + return [] + + with open(filepath, "r") as data: + try: + all_data = json.load(data) + except json.JSONDecodeError: + self.logger.error("%s is not an illegal json file", os.path.basename(filepath)) + return [] + + if isinstance(all_data, list): + build_no_list = [] + for value in all_data: + try: + build_no_list.append(int(value)) + except ValueError as e: + return [] + return build_no_list + else: + self.logger.error("%s content is not a list", os.path.basename(filepath)) + return [] + + @staticmethod + def get_owner_repo(pr_url: str): + """ + get owner and repo from pull request link + :param pr_url: + :return: + """ + res = re.search(r"https://gitee.com/.*/.*/.*/.*", pr_url) + if not res: + raise ValueError("{} format error".format(pr_url)) + sp = re.sub("https://gitee.com/", "", pr_url).split("/") + owner_index = 0 + repo_index = 1 + pr_id_index = 3 + return sp[owner_index], sp[repo_index], sp[pr_id_index] + + @staticmethod + def check_command_format(mistake_comment: str): + """ + analyse ci mistake comment format + :param mistake_comment: + :return: + """ + sp = re.split("\s+", mistake_comment.strip()) + command_index = 0 + build_no_index = 1 + ci_type_stage_index = 2 + + command = sp[command_index] + if any([all([command != "/ci_unmistake", command != "/ci_mistake"]), + all([command == "/ci_unmistake", len(sp) != build_no_index + 1]), + all([command == "/ci_mistake", len(sp) < build_no_index + 1])]): + return False, None, None, None + + try: + build_no = int(sp[build_no_index]) + if command == "/ci_unmistake": + return True, command, build_no, None + except ValueError: + return False, None, None, None + + ci_mistake_type_stage = sp[ci_type_stage_index:] + return True, command, build_no, ci_mistake_type_stage + + def comment_to_pr(self, comment_content): + """ + comment message to pull request + :param comment_content: comment content + :return: + """ + print(self.owner, self.repo, self.gitee_token, self.pr_id) + gp = GiteeProxy(self.owner, self.repo, self.gitee_token) + gp.comment_pr(self.pr_id, comment_content) + + def send_ci_mistake_data(self, command, build_no, ci_mistake_type, ci_mistake_stage): + """ + send message to datebase by kafka + :param command: + :param build_no: + :param ci_mistake_type: + :param ci_mistake_stage: + :return: + """ + message = { + "pr_url": self.pr_url, + "build_no": build_no, + "committer": self.committer, + "commit_at": self.commit_at, + } + if ci_mistake_type: + message["ci_mistake_type"] = ci_mistake_type + if ci_mistake_stage: + message["ci_mistake_stage"] = ci_mistake_stage + + if command == "/ci_unmistake": + message["ci_mistake_status"] = False + else: + message["ci_mistake_status"] = True + + #kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) + #kp.send("openeuler_statewall_ci_mistake", key=self.comment_id, value=message) + self.comment_to_pr("Thanks for your commit.") + self.logger.info("kafka message: %s.", message) + + def process(self, mistake_comment, build_no_filepath): + """ + process to analyse ci mistake content and save to datebase + :param mistake_comment: ci mistake comment + :param build_no_filepath: build number filepath + :return: + """ + build_no_list = self.load_build_no_list(build_no_filepath) + status, command, build_no, ci_mistake_type_stage = CiMistake.check_command_format(mistake_comment) + + if not status: + self.comment_to_pr("comment format error.") + return + + if build_no not in build_no_list: + self.comment_to_pr("**{}** is not an illegal build number. You should select one from **{}**.".format( + build_no, ", ".join([str(item) for item in build_no_list]))) + return + + ci_mistake_type_list = list(set(ci_mistake_type_stage).intersection(set(self.support_mistake_type))) + ci_mistake_stage = list(set(ci_mistake_type_stage).intersection(set(self.support_check_stage))) + ci_mistake_others = list(set(ci_mistake_type_stage).difference( + set(self.support_mistake_type)).difference(set(self.support_check_stage))) + if ci_mistake_others: + self.comment_to_pr("**{}** is not an illegal mistake type or check item. \ + If you want to express mistake type, you can select one from **{}**. \ + If you want to express check item, you can select one or more from **{}**. ".format( + ", ".join(ci_mistake_others), + ", ".join(self.support_mistake_type), + ", ".join(self.support_check_stage))) + return + + if len(ci_mistake_type_list) > 1: + self.comment_to_pr("You should only select one mistake type, now are **{}**. ".format( + ", ".join(ci_mistake_type_list))) + return + ci_mistake_type = ci_mistake_type_list[0] if ci_mistake_type_list else "" + + self.send_ci_mistake_data(command, build_no, ci_mistake_type, ci_mistake_stage) + + +def init_args(): + """ + init args + :return: + """ + parser = argparse.ArgumentParser() + parser.add_argument("--pr_url", type=str, dest="pr_url", help="pull request") + parser.add_argument("--mistake_comment", type=str, dest="mistake_comment", + help="gitee comment for mask ci mistake") + parser.add_argument("--committer", type=str, dest="committer", help="committer") + parser.add_argument("--commit_at", type=str, dest="commit_at", help="commit time") + parser.add_argument("--comment_id", type=str, dest="comment_id", help="comment id") + parser.add_argument("--build_no_filepath", type=str, dest="build_no_filepath", + help="path of a file which record history build numbers") + parser.add_argument("--gitee_token", type=str, dest="gitee_token", help="gitee api token") + + return parser.parse_args() + + +if "__main__" == __name__: + args = init_args() + + ci_mistake = CiMistake(args.pr_url, args.gitee_token, args.committer, args.commit_at, args.comment_id) + ci_mistake.process(args.mistake_comment, args.build_no_filepath) +