From 0017997d722986a1a116d524a21f958987bce2c8 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 9 Aug 2022 15:32:08 +0800 Subject: [PATCH 1/2] ignore modify file check pr rule --- config/config.ini | 2 +- core/check_meta_service.py | 35 ++++++++++++++++++++++------------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/config/config.ini b/config/config.ini index 2f7042a..9b5aadd 100644 --- a/config/config.ini +++ b/config/config.ini @@ -123,6 +123,6 @@ openEuler-22.03-LTS-Epol-Multi-Version-OpenStack-Wallaby = openEuler:22.03:LTS:s # all obs projects path and where store all packages of all obs projects path = /srv/obs/build [obs_ignore_package] -name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv +name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv WasmEngine [obs_include_project] name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Erlang openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby diff --git a/core/check_meta_service.py b/core/check_meta_service.py index 27c522c..539f411 100644 --- a/core/check_meta_service.py +++ b/core/check_meta_service.py @@ -120,6 +120,7 @@ class CheckMetaPull(object): log.info("line:%s" % line) new_file_path = '' complete_new_file_path = '' + modify_file_path = '' log_list = list(line.split()) temp_log_type = log_list[0] if len(log_list) == 3: @@ -133,11 +134,13 @@ class CheckMetaPull(object): new_file_path = list(line.split())[2] complete_new_file_path = list(line.split())[2] elif len(log_list) == 2: + if temp_log_type == "M": + modify_file_path = list(line.split())[1] if temp_log_type != "D": new_file_path = list(line.split())[1] complete_new_file_path = list(line.split())[1] log.info(new_file_path) - return new_file_path, complete_new_file_path + return new_file_path, complete_new_file_path, modify_file_path def _check_file_format(self, file_path): """ @@ -486,14 +489,17 @@ class CheckMetaPull(object): changefile = self._get_change_file() changelist = [] complete_changelist = [] + modify_changelist = [] for msg in changefile: - parse_git, complete_parse_git = self._parse_git_log(msg) + parse_git, complete_parse_git, modify_git = self._parse_git_log(msg) if parse_git: changelist.append(parse_git) if complete_parse_git: complete_changelist.append(complete_parse_git) + if modify_git: + modify_changelist.append(modify_git) if changelist or complete_changelist: - return changelist, complete_changelist + return changelist, complete_changelist, modify_changelist else: log.info("Finish!!There are no file need to check") sys.exit() @@ -670,7 +676,7 @@ class CheckMetaPull(object): os.system("if [ -d community ];then rm -rf community;fi") raise SystemExit("*******community-clone-error:please check your net*******") - def _check_pr_rule(self, release_manage, pr_file): + def _check_pr_rule(self, release_manage, pr_file, modify_pr_file): """ check openeuler_meta pull request """ @@ -679,14 +685,17 @@ class CheckMetaPull(object): failed_msg = [] if release_manage: for change_file in pr_file: - path_info = self._get_path_info(change_file) - if path_info[1] in release_manage: - log.error("check pr rule failed repository path:{}".format(change_file)) - obs_project_name = change_file.split('/')[1] - failed_flag.append('yes') - failed_msg.append(obs_project_name) + if change_file not in modify_pr_file: + path_info = self._get_path_info(change_file) + if path_info[1] in release_manage: + log.error("check pr rule failed repository path:{}".format(change_file)) + obs_project_name = change_file.split('/')[1] + failed_flag.append('yes') + failed_msg.append(obs_project_name) + else: + log.info("check pr rule success repository path:{}".format(change_file)) else: - log.info("check pr rule success repository path:{}".format(change_file)) + log.info("modify file ignore check pr rule repository path:{}".format(change_file)) if failed_flag: log.error("you can not pull request in branch:{},Please refer to this issue:https://gitee.com/openeuler/release-management/issues/I4U2VN?from=project-issue".format(failed_msg)) raise SystemExit("*******PLEASE CHECK YOUR PR*******") @@ -702,8 +711,8 @@ class CheckMetaPull(object): """ if self.prid and self.token: release_management_data = self._release_management_tree() - change_result, complete_change_result = self._get_all_change_file() - pr_check_result = self._check_pr_rule(release_management_data, complete_change_result) + change_result, complete_change_result, modify_result = self._get_all_change_file() + pr_check_result = self._check_pr_rule(release_management_data, complete_change_result, modify_result) self._check_private_sig_pkg(change_result) check_result = self._check_service_meta(change_result) elif not self.prid and self.branch: -- Gitee From 756a4489e92355b1b79de3c3b786e291645f0850 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 10 Aug 2022 14:45:15 +0800 Subject: [PATCH 2/2] asssa --- common/common.py | 143 +++++++++++++++++++++++++++++++ core/check_release_management.py | 62 +++++++++++++- openeuler_obs.py | 8 +- 3 files changed, 211 insertions(+), 2 deletions(-) diff --git a/common/common.py b/common/common.py index b85c223..a55bb35 100644 --- a/common/common.py +++ b/common/common.py @@ -19,6 +19,13 @@ function for all """ import os import pexpect +import requests +import jenkins +from requests.auth import HTTPBasicAuth +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode def str_to_bool(s): @@ -117,6 +124,142 @@ class Pexpect(object): return msg +class Comment(object): + """ + gitee comments process + """ + + def __init__(self, owner, repo, token): + self._owner = owner + self._repo = repo + self._token = token + + + def comment_pr(self, pr, comment): + """ + 评论pull request + :param pr: 本仓库PR id + :param comment: 评论内容 + :return: 0成功，其它失败 + """ + comment_pr_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/comments".format(self._owner, self._repo, pr) + data = {"access_token": self._token, "body": comment} + rs = self.do_requests("post", comment_pr_url, body=data, timeout=10) + if rs != 0: + return False + + def parse_comment_to_table(self, pr, results, tips): + """ + :param pr: 仓库PR id + :param results: 门禁检查返回结果 + :return: none + """ + comment_state = {"success":":white_check_mark:","warning":":bug:","failed":":x:"} + comments = ["",""] + for check_item,check_result in results.items(): + emoji_result = comment_state[check_result] + word_result = check_result.upper() + info_str = "".format(check_item, emoji_result, word_result) + comments.append(info_str) + comments.append("
Check Item Check Result
{} {}{}
") + comments.append(tips) + self.comment_pr(pr, "\n".join(comments)) + + + def do_requests(self, method, url, querystring=None, body=None, auth=None, timeout=30, obj=None): + """ + http request + :param method: http method + :param url: http[s] schema + :param querystring: dict + :param body: json + :param auth: dict, basic auth with user and password + :param timeout: second + :param obj: callback object, support list/dict/object + :return: + """ + try: + if method.lower() not in ["get", "post", "put", "delete"]: + return -1 + if querystring: + url = "{}?{}".format(url, urlencode(querystring)) + func = getattr(requests, method.lower()) + if body: + if auth: + rs = func(url, json=body, timeout=timeout, auth=HTTPBasicAuth(auth["user"], auth["password"])) + else: + rs = func(url, json=body, timeout=timeout) + else: + if auth: + rs = func(url, timeout=timeout, auth=HTTPBasicAuth(auth["user"], auth["password"])) + else: + rs = func(url, timeout=timeout) + if rs.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.no_content]: + return 1 + # return response + if obj is not None: + if isinstance(obj, list): + obj.extend(rs.json()) + elif isinstance(obj, dict): + obj.update(rs.json()) + elif callable(obj): + obj(rs) + elif hasattr(obj, "cb"): + getattr(obj, "cb")(rs.json()) + return 0 + except requests.exceptions.SSLError as e: + return -2 + except requests.exceptions.Timeout as e: + return 2 + except requests.exceptions.RequestException as e: + return 3 + +class JenkinsProxy(object): + """ + Jenkins 代理，实现jenkins一些操作 + """ + + def __init__(self, base_url, username, token, timeout=10): + """ + + :param base_url: + :param username: 用户名 + :param token: + :param timeout: + """ + self._username = username + self._token = token + self._timeout = timeout + self._jenkins = jenkins.Jenkins(base_url, username=username, password=token, timeout=timeout) + + def get_job_info(self, job_path): + """ + 获取任务信息 + :param job_path: job路径 + :return: None if job_path not exist + """ + try: + return self._jenkins.get_job_info(job_path) + except jenkins.JenkinsException as e: + return None + + @classmethod + def get_job_path_from_job_url(cls, job_url): + """ + 从url中解析job路径 + :param job_url: 当前工程url, for example https://domain/job/A/job/B/job/C + :return: for example, A/B/C + """ + jenkins_first_level_dir_index = 2 + jenkins_dir_interval_with_level = 2 + job_path = re.sub(r"/$", "", job_url) + job_path = re.sub(r"http[s]?://", "", job_path) + sp = job_path.split("/")[jenkins_first_level_dir_index:: + jenkins_dir_interval_with_level] + sp = [item for item in sp if item != ""] + job_path = "/".join(sp) + return job_path + if __name__ == "__main__": res = git_repo_src("https://gitee.com/src-openeuler/zip", "xxxxx", "xxxxx") diff --git a/core/check_release_management.py b/core/check_release_management.py index 8cdff62..751300d 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -26,6 +26,8 @@ sys.path.append(os.path.join(Now_path, "..")) from common.log_obs import log from collections import Counter from common.common import git_repo_src +from common.common import Comment +from common.common import JenkinsProxy class CheckReleaseManagement(object): """ @@ -40,11 +42,15 @@ class CheckReleaseManagement(object): """ self.kwargs = kwargs self.prid = self.kwargs['pr_id'] + self.token = self.kwargs['access_token'] self.current_path = os.getcwd() self.meta_path = self.kwargs['obs_meta_path'] self.manage_path = self.kwargs['release_management_path'] self.giteeuser = self.kwargs['gitee_user'] self.giteeuserpwd = self.kwargs['gitee_pwd'] + self.jenkins_user = self.kwargs['jenkins_user'] + self.jenkins_api_token = self.kwargs['jenkins_api_token'] + self.job_result = {'check_yaml_format':'success','check_package_complete':'success','check_package_add':'success','check_package_move':'success','check_package_delete':'success','check_date':'success'} def _clean(self, pkgname): """ @@ -334,8 +340,10 @@ class CheckReleaseManagement(object): result = yaml.load(f, Loader=yaml.FullLoader) log.info("{0} format check".format(yaml_path)) except Exception as e: + self.job_result['check_yaml_format'] = 'failed' log.error("**********FORMAT ERROR***********") log.error("%s format bad Because:%s" % (yaml_path, e)) + self._comment_to_pr() raise SystemExit("May be %s has a bad format" % yaml_path) def _check_same_pckg(self, change_file_path, yaml_msg): @@ -488,6 +496,7 @@ class CheckReleaseManagement(object): else: error_infos[c_branch] = [pkg] if error_infos: + self.job_result['check_package_add'] = 'failed' log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag @@ -512,6 +521,7 @@ class CheckReleaseManagement(object): add_names.remove(name) if add_names: error_flag = True + self.job_result['check_package_delete'] = 'failed' log.error("master branch pkg name:{} you want delete not exist in obs_meta!!!".format(add_names)) return error_flag @@ -584,6 +594,7 @@ class CheckReleaseManagement(object): else: error_infos[branch] = [pkg] if error_infos: + self.job_result['check_package_move'] = 'failed' log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag @@ -610,6 +621,7 @@ class CheckReleaseManagement(object): else: error_infos[branch] = [pkg] if error_infos: + self.job_result['check_date'] = 'failed' log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag @@ -631,11 +643,14 @@ class CheckReleaseManagement(object): error_master_pkgs = list(set(old_pkgs).difference(set(pkgs))) if error_master_pkgs: error_flag = True + self.job_result['check_package_complete'] = 'failed' log.error("The following {0} packages should not deleted in the master YAML files".format(error_master_pkgs)) if duplicated: error_flag = True + self.job_result['check_package_complete'] = 'failed' log.error("The following {0} packages are duplicated in the master YAML files".format(duplicated)) if error_flag: + self._comment_to_pr() raise SystemExit("ERROR: Please check your PR") def _check_pkg_from_new(self, meta_path, change_info): @@ -674,6 +689,7 @@ class CheckReleaseManagement(object): pkgs.remove(name) if pkgs: log.error("The {0} not exist in obs_meta dir {1}".format(pkgs,branch)) + self.job_result['check_package_delete'] = 'failed' error_flag = True return error_flag @@ -714,6 +730,7 @@ class CheckReleaseManagement(object): break if error_names: error_flag =True + self.job_result['check_package_add'] = 'failed' for pkg in pkgs: if pkg['name'] in error_names: log.error("branch:{}:The {} not exist in obs_meta from dir {}/{}".format(branch, pkg['name'], pkg['source_dir'], pkg['obs_from'])) @@ -776,6 +793,7 @@ class CheckReleaseManagement(object): yaml_date = int(pkg['date'].split('-')[2]) if today != yaml_date: error_flag = True + self.job_result['check_date'] = 'failed' log.error("Wrong Date: !!!".format(pkg['date'])) return error_flag @@ -852,9 +870,13 @@ class CheckReleaseManagement(object): log.info(pkg) if error_pkg_flag: log.error("May be {0} should not be delete".format(error_pkg)) + self.job_result['check_package_complete'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please check your PR") if same_pkg_flag: log.error("The following {0} packages are duplicated in the YAML files".format(same_pkg)) + self.job_result['check_package_complete'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please check your PR") return change_infos @@ -885,6 +907,8 @@ class CheckReleaseManagement(object): if new_source != new_destination or new_obsfrom != old_obsto: error_flag = True log.error("{}:{}".format(pkgname, obsinfo)) + self.job_result['check_package_move'] = 'failed' + self._comment_to_pr() log.error("internal move pkg:{} source_dir must same with destination_dir and obs_from must same with before obs_to".format(pkgname)) if error_flag: raise SystemExit("ERROR: Please check your PR") @@ -958,10 +982,43 @@ class CheckReleaseManagement(object): all_master_pkgs = self._get_complete_yaml_pkgs('master') return all_master_pkgs + def _comment_to_pr(self): + """ + gitee comment and jenkins api comment check result to pr + """ + jp = JenkinsProxy("https://openeulerjenkins.osinfra.cn/", self.jenkins_user, self.jenkins_api_token) + trigger_job_name = os.environ.get("JOB_NAME") + trigger_build_id = os.environ.get("BUILD_ID") + log.info("trigger_job_name---------------------------------------------------------") + log.info(trigger_job_name) + log.info(trigger_build_id) + trigger_job_info = jp.get_job_info(trigger_job_name) + trigger_job_url = trigger_job_info.get("url") + gm = Comment('dong-jie_1','release-management',self.token) + comment_tips = "1)若您对如何创建提交PR有疑问，" \ + "可参考" \ + "开发者提交PR指导手册\n" + gm.parse_comment_to_table(self.prid, self.job_result, comment_tips) + + # def _comment_jenkins_url(gp, jp): + # """ + # 在pr评论中显示构建任务链接 + # :param jp: jenkins接口 + # :return: + # """ + # trigger_job_name = os.environ.get("JOB_NAME") + # trigger_build_id = os.environ.get("BUILD_ID") + # trigger_job_info = jp.get_job_info(trigger_job_name) + # trigger_job_url = trigger_job_info.get("url") + # comments.append("门禁入口及编码规范检查: {}, 当前构建号为 {}".format( + # trigger_job_url, jp.get_job_path_from_job_url(trigger_job_url), trigger_build_id)) + # gp.comment_pr(pr, "\n".join(comments)) + def check_pckg_yaml(self): """ check the obs_from branch_from in pckg-mgmt.yaml """ + # job_result = {'check_yaml_format':'','check_package_complete':'','check_package_add':'','check_package_move':'','check_package_delete':'','check_date':''} change = self._get_repo_change_file('openeuler', 'release-management', self.manage_path) change_file,master_change_file,new_version_change_file = self._parse_commit_file(change) @@ -988,6 +1045,7 @@ class CheckReleaseManagement(object): del_flag = self._check_master_del_rules(del_old_master_yaml_msg, del_master_change_yaml_msg) date_flag = self._check_master_date_rules(add_infos) if add_flag or move_flag or date_flag or del_flag: + self._comment_to_pr() raise SystemExit("Please check your commit") if new_version_change_file: log.info(new_version_change_file) @@ -1001,6 +1059,7 @@ class CheckReleaseManagement(object): error_flag_add = self._check_pkg_parent_from(change_infos, correct_from, error_from, add_infos) error_flag_del = self._check_pkg_delete_new(self.meta_path, change_delete_infos) if error_flag_add or error_flag_del or date_flag: + self._comment_to_pr() raise SystemExit("Please check your commit") if change_file: log.info(change_file) @@ -1014,9 +1073,10 @@ class CheckReleaseManagement(object): error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) if error_flag1 or error_flag2 or error_flag3 or error_flag4: raise SystemExit("Please check your commit") + self._comment_to_pr() if __name__ == "__main__": - kw = {"branch":"master", + kw = {"branch":"master", "gitee_user":"", "gitee_pwd":"", "pr_id":"", diff --git a/openeuler_obs.py b/openeuler_obs.py index 30328c0..2db7588 100644 --- a/openeuler_obs.py +++ b/openeuler_obs.py @@ -113,6 +113,9 @@ par.add_argument("-a", "--ALL_", default=False, help="update all obs repo rpms", par.add_argument("-cmp", "--compare", default=False, help="compare rpm", required=False) par.add_argument("-cpm", "--check_pckg_mgmt", default=False, help="check if there are any problems with the commi for release-management", required=False) +par.add_argument("-jl", "--jenkins_base_url", default="https://openeulerjenkins.osinfra.cn/", help="jenkins base url") +par.add_argument("-ju", "--jenkins_user", help="jekins user name") +par.add_argument("-jt", "--jenkins_api_token", help="jenkins api token") args = par.parse_args() #apply @@ -158,7 +161,10 @@ kw = { "pckg_mgmt": args.pckg_mgmt, "release_management_path": args.remt, "check_pckg_mgmt": args.check_pckg_mgmt, - "compare": args.compare + "compare": args.compare, + "jenkins_base_url": args.jenkins_base_url, + "jenkins_user": args.jenkins_user, + "jenkins_api_token": args.jenkins_api_token } run = Runner(**kw) -- Gitee