From 4d03ec72e0795f9c979c496d3b5426c3920e6b53 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Thu, 18 Aug 2022 11:12:45 +0800 Subject: [PATCH] add gitee comment and jenkins api,add check pkg requires --- common/common.py | 162 +++++++++++++++++++++++++++++++ common/parser_config.py | 15 +++ config/config.ini | 2 + core/check_release_management.py | 148 ++++++++++++++++++++++++++-- core/sync_pckg_mgmt.py | 50 +++++----- openeuler_obs.py | 8 +- 6 files changed, 354 insertions(+), 31 deletions(-) diff --git a/common/common.py b/common/common.py index b85c223..8d6391c 100644 --- a/common/common.py +++ b/common/common.py @@ -18,7 +18,15 @@ function for all """ import os +import re import pexpect +import requests +import jenkins +from requests.auth import HTTPBasicAuth +try: + from urllib import urlencode +except ImportError: + from urllib.parse import urlencode def str_to_bool(s): @@ -117,6 +125,160 @@ class Pexpect(object): return msg +class Comment(object): + """ + gitee comments process + :param owner: 仓库属于哪个组织 + :param repo: 仓库名 + :param token: gitee 账户token + """ + + def __init__(self, owner, repo, token): + self._owner = owner + self._repo = repo + self._token = token + + + def comment_pr(self, pr, comment): + """ + 评论pull request + :param pr: 本仓库PR id + :param comment: 评论内容 + :return: 0成功，其它失败 + """ + comment_pr_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/comments".format(self._owner, self._repo, pr) + data = {"access_token": self._token, "body": comment} + rs = self.do_requests("post", comment_pr_url, body=data, timeout=10) + if rs == 0: + return True + else: + return False + + def parse_comment_to_table(self, pr, results, tips, details): + """ + :param pr: 仓库PR id + :param results: 门禁检查返回结果 + :return: none + """ + comment_state = {"success":":white_check_mark:", "warning":":bug:", "failed":":x:"} + comments = ["", ""] + for check_item, check_result in results.items(): + emoji_result = comment_state[check_result] + word_result = check_result.upper() + info_str = ''''''.format(check_item, emoji_result, word_result, details[check_item]) + comments.append(info_str) + comments.append("
Check Item Check Result Description
{} {}{} {}
") + comments.extend(tips) + self.comment_pr(pr, "\n".join(comments)) + + + def do_requests(self, method, url, querystring=None, body=None, auth=None, timeout=30, obj=None): + """ + http request + :param method: http method + :param url: http[s] schema + :param querystring: dict + :param body: json + :param auth: dict, basic auth with user and password + :param timeout: second + :param obj: callback object, support list/dict/object + :return: + """ + try: + if method.lower() not in ["get", "post", "put", "delete"]: + return -1 + if querystring: + url = "{}?{}".format(url, urlencode(querystring)) + func = getattr(requests, method.lower()) + if body: + if auth: + rs = func(url, json=body, timeout=timeout, auth=HTTPBasicAuth(auth["user"], auth["password"])) + else: + rs = func(url, json=body, timeout=timeout) + else: + if auth: + rs = func(url, timeout=timeout, auth=HTTPBasicAuth(auth["user"], auth["password"])) + else: + rs = func(url, timeout=timeout) + if rs.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.no_content]: + return 1 + # return response + if obj is not None: + if isinstance(obj, list): + obj.extend(rs.json()) + elif isinstance(obj, dict): + obj.update(rs.json()) + elif callable(obj): + obj(rs) + elif hasattr(obj, "cb"): + getattr(obj, "cb")(rs.json()) + return 0 + except requests.exceptions.SSLError as e: + return -2 + except requests.exceptions.Timeout as e: + return 2 + except requests.exceptions.RequestException as e: + return 3 + +class JenkinsProxy(object): + """ + Jenkins 代理，实现jenkins一些操作 + """ + + def __init__(self, base_url, username, token, timeout=10): + """ + + :param base_url: + :param username: 用户名 + :param token: + :param timeout: + """ + self._username = username + self._token = token + self._timeout = timeout + self._jenkins = jenkins.Jenkins(base_url, username=username, password=token, timeout=timeout) + + def get_job_info(self, job_path): + """ + 获取任务信息 + :param job_path: job路径 + :return: None if job_path not exist + """ + try: + return self._jenkins.get_job_info(job_path) + except jenkins.JenkinsException as e: + return None + + @classmethod + def get_job_path_from_job_url(cls, job_url): + """ + 从url中解析job路径 + :param job_url: 当前工程url, for example https://domain/job/A/job/B/job/C + :return: for example, A/B/C + """ + jenkins_first_level_dir_index = 2 + jenkins_dir_interval_with_level = 2 + job_path = re.sub(r"/$", "", job_url) + job_path = re.sub(r"http[s]?://", "", job_path) + sp = job_path.split("/")[jenkins_first_level_dir_index:: + jenkins_dir_interval_with_level] + sp = [item for item in sp if item != ""] + job_path = "/".join(sp) + return job_path + + @staticmethod + def get_job_path_build_no_from_build_url(build_url): + """ + 从url中解析job路径 + :param build_url: 当前构建url, for example https://domain/job/A/job/B/job/C/number/ + :return: for example A/B/C/number + """ + job_build_no = re.sub(r"/$", "", build_url) + job_url = os.path.dirname(job_build_no) + build_no = os.path.basename(job_build_no) + job_path = JenkinsProxy.get_job_path_from_job_url(job_url) + return job_path, build_no + if __name__ == "__main__": res = git_repo_src("https://gitee.com/src-openeuler/zip", "xxxxx", "xxxxx") diff --git a/common/parser_config.py b/common/parser_config.py index b4fcd4d..954a228 100644 --- a/common/parser_config.py +++ b/common/parser_config.py @@ -68,6 +68,7 @@ class ParserConfigIni(object): self._init_obs_prj_root_path() self._init_obs_ignored_package() self._init_obs_include_project() + self._init_error_from_project() def _init_branch_list(self): """ @@ -140,6 +141,20 @@ class ParserConfigIni(object): """ return self.obs_include_projects + def _init_error_from_project(self): + """ + init include project list + return: None + """ + self.error_from_projects = self.config.get("error_from_project", "name").split(" ") + + def get_error_from_project(self): + """ + get error from projects + return: error from project list + """ + return self.error_from_projects + def _init_package_info_file(self): """ init package info file for store package which is not be updated diff --git a/config/config.ini b/config/config.ini index 9b5aadd..3133e75 100644 --- a/config/config.ini +++ b/config/config.ini @@ -126,3 +126,5 @@ path = /srv/obs/build name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv WasmEngine [obs_include_project] name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Erlang openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby +[error_from_project] +name = openEuler:Factory diff --git a/core/check_release_management.py b/core/check_release_management.py index 8cdff62..5d59ca7 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -17,6 +17,7 @@ check the software package for the corresponding project of thecorresponding branch of source """ import os +import re import sys import yaml import requests @@ -26,6 +27,9 @@ sys.path.append(os.path.join(Now_path, "..")) from common.log_obs import log from collections import Counter from common.common import git_repo_src +from common.common import Comment +from common.common import JenkinsProxy +from common.parser_config import ParserConfigIni class CheckReleaseManagement(object): """ @@ -40,11 +44,18 @@ class CheckReleaseManagement(object): """ self.kwargs = kwargs self.prid = self.kwargs['pr_id'] + self.token = self.kwargs['access_token'] self.current_path = os.getcwd() self.meta_path = self.kwargs['obs_meta_path'] self.manage_path = self.kwargs['release_management_path'] self.giteeuser = self.kwargs['gitee_user'] self.giteeuserpwd = self.kwargs['gitee_pwd'] + self.jenkins_user = self.kwargs['jenkins_user'] + self.jenkins_api_token = self.kwargs['jenkins_api_token'] + self.jenkins_build_url = self.kwargs['jenkins_build_url'] + self.job_result = {'check_yaml_format':'success','check_package_yaml_key':'success','check_package_complete':'success','check_package_requires':'success','check_package_add':'success','check_package_move':'success','check_package_delete':'success','valied_package_source':'success','check_date':'success'} + par = ParserConfigIni() + self.error_from_project = par.get_error_from_project() def _clean(self, pkgname): """ @@ -213,6 +224,8 @@ class CheckReleaseManagement(object): del error_pkg[change_file] if error_pkg: log.error("May be {0} should not be delete".format(error_pkg)) + self.job_result['check_package_complete'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please check your PR") def _check_key_in_yaml(self, all_pack_msg, change_file): @@ -234,6 +247,8 @@ class CheckReleaseManagement(object): error_flag = True log.error("Please check {0}".format(msg)) if error_flag: + self.job_result['check_package_yaml_key'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please ensure the following key values in your yaml") def _check_date_time(self, yaml_msg, change_file): @@ -252,6 +267,7 @@ class CheckReleaseManagement(object): log.error(msg) log.error("Wrong Date: !!!".format(msg['date'])) if error_flag: + self.job_result['check_date'] = 'failed' log.error("Please set your date to the same day as the commit time!!!") return error_flag @@ -271,6 +287,7 @@ class CheckReleaseManagement(object): yaml_key = os.path.join(msg['branch_from'], msg['obs_from'], msg['name']) log.error("The {0} not exist in obs_meta".format(yaml_key)) + self.job_result['check_package_add'] = 'failed' error_flag = True return error_flag @@ -321,6 +338,7 @@ class CheckReleaseManagement(object): return change_list else: log.info("The are no new msg in your yaml!!!") + self._comment_to_pr() sys.exit() def _check_yaml_format(self, yaml_path_list, manage_path): @@ -334,8 +352,10 @@ class CheckReleaseManagement(object): result = yaml.load(f, Loader=yaml.FullLoader) log.info("{0} format check".format(yaml_path)) except Exception as e: + self.job_result['check_yaml_format'] = 'failed' log.error("**********FORMAT ERROR***********") log.error("%s format bad Because:%s" % (yaml_path, e)) + self._comment_to_pr() raise SystemExit("May be %s has a bad format" % yaml_path) def _check_same_pckg(self, change_file_path, yaml_msg): @@ -355,6 +375,7 @@ class CheckReleaseManagement(object): if all_pkg_name: log.error("The following packages are duplicated in the YAML files") log.error(all_pkg_name) + self.job_result['check_package_complete'] = 'failed' return True else: return False @@ -385,6 +406,7 @@ class CheckReleaseManagement(object): for msg in error_msg[yaml_path]: log.error(msg) if error_msg: + self.job_result['valied_package_source'] = 'failed' return True else: return False @@ -421,6 +443,8 @@ class CheckReleaseManagement(object): del info_dict[change] if error_flag: log.error("Check the delete group in the {0}!!!".format(info_dict)) + self.job_result['check_package_delete'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR:Please check your PR") def _get_move_and_add(self,old_msg,new_msg): @@ -480,6 +504,12 @@ class CheckReleaseManagement(object): error_infos[c_branch] = [pkg] else: c_branch = branch.replace("-",":") + error_flag = True + log.error("pkg name:{0} must have obs_from in !!!".format(pkg['name'], branch)) + if error_infos.get(c_branch,[]): + error_infos[c_branch].append(pkg) + else: + error_infos[c_branch] = [pkg] if pkg['obs_to'] != c_branch: error_flag = True log.error("pkg name:{2} Wrong obs_to: in !!!".format(pkg['obs_to'],branch,pkg['name'])) @@ -488,6 +518,7 @@ class CheckReleaseManagement(object): else: error_infos[c_branch] = [pkg] if error_infos: + self.job_result['check_package_add'] = 'failed' log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag @@ -512,6 +543,7 @@ class CheckReleaseManagement(object): add_names.remove(name) if add_names: error_flag = True + self.job_result['check_package_delete'] = 'failed' log.error("master branch pkg name:{} you want delete not exist in obs_meta!!!".format(add_names)) return error_flag @@ -584,6 +616,7 @@ class CheckReleaseManagement(object): else: error_infos[branch] = [pkg] if error_infos: + self.job_result['check_package_move'] = 'failed' log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag @@ -610,6 +643,7 @@ class CheckReleaseManagement(object): else: error_infos[branch] = [pkg] if error_infos: + self.job_result['check_date'] = 'failed' log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag @@ -631,11 +665,14 @@ class CheckReleaseManagement(object): error_master_pkgs = list(set(old_pkgs).difference(set(pkgs))) if error_master_pkgs: error_flag = True + self.job_result['check_package_complete'] = 'failed' log.error("The following {0} packages should not deleted in the master YAML files".format(error_master_pkgs)) if duplicated: error_flag = True + self.job_result['check_package_complete'] = 'failed' log.error("The following {0} packages are duplicated in the master YAML files".format(duplicated)) if error_flag: + self._comment_to_pr() raise SystemExit("ERROR: Please check your PR") def _check_pkg_from_new(self, meta_path, change_info): @@ -674,6 +711,7 @@ class CheckReleaseManagement(object): pkgs.remove(name) if pkgs: log.error("The {0} not exist in obs_meta dir {1}".format(pkgs,branch)) + self.job_result['check_package_delete'] = 'failed' error_flag = True return error_flag @@ -714,6 +752,7 @@ class CheckReleaseManagement(object): break if error_names: error_flag =True + self.job_result['check_package_add'] = 'failed' for pkg in pkgs: if pkg['name'] in error_names: log.error("branch:{}:The {} not exist in obs_meta from dir {}/{}".format(branch, pkg['name'], pkg['source_dir'], pkg['obs_from'])) @@ -740,6 +779,8 @@ class CheckReleaseManagement(object): error_flag = True log.error("Please check {0}".format(msg)) if error_flag: + self.job_result['check_package_yaml_key'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please ensure the following key values in your yaml") def _check_valid_release_branch(self, change_info): @@ -755,12 +796,15 @@ class CheckReleaseManagement(object): for pkg in msg: if pkg['destination_dir'] == branch and \ pkg['source_dir'] in branch_result['branch'].keys() and \ - pkg['destination_dir'] in branch_result['branch'][pkg['source_dir']]: + pkg['destination_dir'] in branch_result['branch'][pkg['source_dir']] and \ + pkg['obs_from'] not in self.error_from_project: continue else: error_flag = True - log.error("pkg:{} souce_dir or destination_dir valid check error".format(pkg['name'])) + log.error("pkg:{} souce_dir/destination_dir/obs_from valid check error".format(pkg['name'])) if error_flag: + self.job_result['valied_package_source'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please ensure the source_dir and destination_dir adapt rules") def _check_pkg_date(self, change_info): @@ -776,6 +820,7 @@ class CheckReleaseManagement(object): yaml_date = int(pkg['date'].split('-')[2]) if today != yaml_date: error_flag = True + self.job_result['check_date'] = 'failed' log.error("Wrong Date: !!!".format(pkg['date'])) return error_flag @@ -852,9 +897,13 @@ class CheckReleaseManagement(object): log.info(pkg) if error_pkg_flag: log.error("May be {0} should not be delete".format(error_pkg)) + self.job_result['check_package_complete'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please check your PR") if same_pkg_flag: log.error("The following {0} packages are duplicated in the YAML files".format(same_pkg)) + self.job_result['check_package_complete'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please check your PR") return change_infos @@ -864,15 +913,16 @@ class CheckReleaseManagement(object): ''' log.info("internal move pkgs check") error_flag = False + internal_move_pkgs = {} for branch,new_msgs in new_msg.items(): if old_msg.get(branch, []): temp_new = {} temp_old = {} old_msgs = old_msg[branch] for new_pkg in new_msgs: - temp_new[new_pkg['name']] = {'obs_to':new_pkg['obs_to'],'obs_from':new_pkg['obs_from'],'source_dir':new_pkg['source_dir'],'destination_dir':new_pkg['destination_dir']} + temp_new[new_pkg['name']] = {'name':new_pkg['name'],'obs_to':new_pkg['obs_to'],'obs_from':new_pkg['obs_from'],'source_dir':new_pkg['source_dir'],'destination_dir':new_pkg['destination_dir']} for old_pkg in old_msgs: - temp_old[old_pkg['name']] = {'obs_to':old_pkg['obs_to'],'obs_from':old_pkg['obs_from'],'source_dir':old_pkg['source_dir'],'destination_dir':old_pkg['destination_dir']} + temp_old[old_pkg['name']] = {'name':old_pkg['name'],'obs_to':old_pkg['obs_to'],'obs_from':old_pkg['obs_from'],'source_dir':old_pkg['source_dir'],'destination_dir':old_pkg['destination_dir']} for pkgname,obsinfo in temp_new.items(): if temp_old.get(pkgname,''): old_obsto = temp_old[pkgname]['obs_to'] @@ -886,7 +936,16 @@ class CheckReleaseManagement(object): error_flag = True log.error("{}:{}".format(pkgname, obsinfo)) log.error("internal move pkg:{} source_dir must same with destination_dir and obs_from must same with before obs_to".format(pkgname)) + else: + if internal_move_pkgs.get(branch, []): + internal_move_pkgs[branch].append(obsinfo) + else: + internal_move_pkgs[branch] = [obsinfo] + if internal_move_pkgs: + self._check_move_pkg_depends(internal_move_pkgs) if error_flag: + self.job_result['check_package_move'] = 'failed' + self._comment_to_pr() raise SystemExit("ERROR: Please check your PR") def _get_new_version_yaml_msg(self, yaml_path_list, manage_path,vtype='master'): @@ -958,6 +1017,79 @@ class CheckReleaseManagement(object): all_master_pkgs = self._get_complete_yaml_pkgs('master') return all_master_pkgs + def _check_move_pkg_depends(self, add_infos): + """ + check move pkgs depends pkgs + """ + for branch,pkgs in add_infos.items(): + log.info('check branch:{} pkgs depends check running...'.format(branch)) + for pkg in pkgs: + if pkg['obs_from'] and 'Multi-Version' not in branch: + project = pkg['obs_from'] + self.get_pkg_depends(project, pkg) + + def get_pkg_depends(self, project, pkg): + """ + :param project:obs project name + :param pkgs: internal move pkgs name + :return none + """ + architecture = ['x86_64','aarch64'] + packages = list(set(os.popen("osc list {}".format(project)).read().split("\n")) - set([])) + pkg = pkg['name'] + for arch in architecture: + temp_path = os.path.join(os.getcwd(),'temp_pkgs', arch, project, pkg) + if not os.path.exists(temp_path): + os.makedirs(temp_path) + res=os.system("cd {0} && osc getbinaries {1} {2} standard_{3} {3}".format(temp_path, project, pkg, arch)) + binaries_path = os.path.join(temp_path, 'binaries') + if os.path.exists(binaries_path): + cmd = '''cd {} && rpm -qp --provides *.rpm|awk '{{print$1}}'|for line in `xargs`;do dnf repoquery --whatrequires "$line";done | tee log'''.format(binaries_path) + rpm_provides = [x.strip() for x in list(set((os.popen(cmd)).read().split("\n"))) if x.strip() != ''] + rpm_provides_name = list(map(self.rpm_name, rpm_provides)) + require_rpms = list(set(rpm_provides_name).intersection(set(packages))) + if require_rpms: + self.job_result['check_package_requires'] = 'warning' + log.warning("{} as follow pkgs {} in project {} are require by pkg {}!!!".format(arch, require_rpms, project, pkg)) + + def rpm_name(self, rpm): + """ + :param rpm:complete rpm name + :return:only rpm name + """ + m = re.match(r"^(.+)-.+-.+", rpm) + if m: + return m.group(1) + else: + return rpm + + def _comment_to_pr(self): + """ + gitee comment and jenkins api comment check result to pr + """ + comment_tips = [] + jp = JenkinsProxy("https://openeulerjenkins.osinfra.cn/", self.jenkins_user, self.jenkins_api_token) + build_url = self.jenkins_build_url + job_name, job_id = jp.get_job_path_build_no_from_build_url(build_url) + comment_tips.append("1)本次构建号为{2}/{1}，点击可查看构建详情: #{1}\n".format(build_url,job_id,job_name)) + comment_tips.append("2)若有检查失败项目，请勿合入") + comment_tips.append("3)若您对如何创建提交PR有疑问，" \ + "可参考" \ + "开发者提交PR指导手册") + details = { + 'check_yaml_format':"检查yaml格式是否正确", + 'check_package_yaml_key':"检查yaml中包key填写是否正确", + 'check_package_complete':"检查是否误删包或者增加不应添加的包", + 'check_package_requires':"检查分支内移动包是否被工程内其他包所依赖", + 'check_package_add':"检查向工程内添加包是否符合规则", + 'check_package_move':"检查分支内包移动是否符合规则", + 'check_package_delete':"检查删除包是否符合规则", + 'valied_package_source':"检查包引入时的引入分支是否符合规则", + 'check_date':"日期检查,必须与提交日期一致" + } + gm = Comment('openeuler','release-management',self.token) + gm.parse_comment_to_table(self.prid, self.job_result, comment_tips, details) + def check_pckg_yaml(self): """ check the obs_from branch_from in pckg-mgmt.yaml @@ -988,19 +1120,21 @@ class CheckReleaseManagement(object): del_flag = self._check_master_del_rules(del_old_master_yaml_msg, del_master_change_yaml_msg) date_flag = self._check_master_date_rules(add_infos) if add_flag or move_flag or date_flag or del_flag: + self._comment_to_pr() raise SystemExit("Please check your commit") if new_version_change_file: log.info(new_version_change_file) change_infos = self._check_rpms_complete_and_repeat(old_new_version_msg, new_version_change_msg) change_delete_infos = self._ensure_delete_infos(del_old_new_version_msg, del_new_version_change_msg) - self._check_internal_move(old_new_version_msg, new_version_change_msg) self._check_key_in_yaml_new(change_infos) self._check_valid_release_branch(change_infos) + self._check_internal_move(old_new_version_msg, new_version_change_msg) date_flag = self._check_pkg_date(change_infos) correct_from, error_from = self._check_pkg_from_new(self.meta_path, change_infos) error_flag_add = self._check_pkg_parent_from(change_infos, correct_from, error_from, add_infos) error_flag_del = self._check_pkg_delete_new(self.meta_path, change_delete_infos) if error_flag_add or error_flag_del or date_flag: + self._comment_to_pr() raise SystemExit("Please check your commit") if change_file: log.info(change_file) @@ -1013,10 +1147,12 @@ class CheckReleaseManagement(object): error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) if error_flag1 or error_flag2 or error_flag3 or error_flag4: + self._comment_to_pr() raise SystemExit("Please check your commit") + self._comment_to_pr() if __name__ == "__main__": - kw = {"branch":"master", + kw = {"branch":"master", "gitee_user":"", "gitee_pwd":"", "pr_id":"", diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 9970b9b..297520b 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -324,11 +324,11 @@ class SyncPckgMgmt(object): log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'], tmp['obs_from'], tmp['obs_to'])) return success_pkg_name - def _add_master_pkg_service(self, package): + def _add_master_pkg_service(self, package, project): """ create and add master service to repo obs_meta """ - filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) + filepath = os.path.join(self.obs_meta_path, 'master', project, package) if not os.path.exists(filepath): os.makedirs(filepath) file_msg = """ @@ -341,12 +341,12 @@ class SyncPckgMgmt(object): """.format(package) try: - with open(os.path.join(filepath,'_service'),'w') as f: + with open(os.path.join(filepath,'_service'), 'w') as f: f.write(file_msg) - log.info("add openEuler:Factory {} _service success!".format(package)) + log.info("add {} {} _service success!".format(project, package)) except Exception as e: log.error(e) - log.error("add openEuler:Factory {} _service failed!".format(package)) + log.error("add {} {} _service failed!".format(project, package)) def _move_master_pkg_service(self, msg): """ @@ -362,7 +362,7 @@ class SyncPckgMgmt(object): else: pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) if not os.path.exists(pkg_path): - self._add_master_pkg_service(tmp['pkgname']) + self._add_master_pkg_service(tmp['pkgname'], tmp['obs_to']) change_pkgs.append(tmp['pkgname']) else: pkg_name = self._move_pkg_service(tmp) @@ -480,10 +480,7 @@ class SyncPckgMgmt(object): meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) if 'README.md' in meta_pkglist: meta_pkglist.remove('README.md') - if len(list(prj_pkg.keys())) >= 3: - need_del_pkg = [] - else: - need_del_pkg = set(meta_pkglist) - set(pkg) + need_del_pkg = set(meta_pkglist) - set(pkg) log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) if need_del_pkg: for del_pkg in need_del_pkg: @@ -508,7 +505,7 @@ class SyncPckgMgmt(object): release_path = os.path.join(self.release_management_path, branch, c_dir, 'pckg-mgmt.yaml') if os.path.exists(release_path): with open(release_path, 'r', encoding='utf-8') as f: - result = yaml.load(f, Loader=yaml.FullLoader) + result = yaml.load(f, Loader=yaml.FullLoader) all_branch_pkgs.extend(result['packages']) for pkg in all_branch_pkgs: project_pkgs.setdefault(pkg['obs_to'], []).append(pkg['name']) @@ -524,18 +521,22 @@ class SyncPckgMgmt(object): self._del_pkg(tmp) if need_add_pkg: temp_lists = [] - for pkg_all in all_branch_pkgs: - if pkg_all['name'] in list(need_add_pkg): - temp_dict = { - 'pkgname':pkg_all['name'], - 'branch_from':pkg_all['source_dir'], - 'branch_to':pkg_all['destination_dir'], - 'obs_from':pkg_all['obs_from'], - 'obs_to':pkg_all['obs_to'], - } - temp_lists.append(temp_dict) - for temp in temp_lists: - self._add_pkg_service(temp) + if branch != 'master': + for pkg_all in all_branch_pkgs: + if pkg_all['name'] in list(need_add_pkg): + temp_dict = { + 'pkgname':pkg_all['name'], + 'branch_from':pkg_all['source_dir'], + 'branch_to':pkg_all['destination_dir'], + 'obs_from':pkg_all['obs_from'], + 'obs_to':pkg_all['obs_to'], + } + temp_lists.append(temp_dict) + for temp in temp_lists: + self._add_pkg_service(temp) + else: + for pkgname in need_add_pkg: + self._add_master_pkg_service(pkgname, project) def _push_code(self, repo): @@ -613,6 +614,7 @@ class SyncPckgMgmt(object): branch_infos = name.split('/') if 'master' in branch_infos: branch = branch_infos[1] + change_branch_lists.append('master') if branch != 'delete': msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) @@ -621,7 +623,7 @@ class SyncPckgMgmt(object): else: msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") pkgs = [tmp['pkgname'] for tmp in msg] - del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs, '{}'.format(branch_infos[0])) + del_change_pkgs, yaml_changes = self._del_pkg_new(pkgs, '{}'.format(branch_infos[0])) self._write_release_yaml(del_change_pkgs, branch_infos[0]) self._del_pckg_from_yaml(yaml_changes) elif 'multi_version' in branch_infos: diff --git a/openeuler_obs.py b/openeuler_obs.py index 30328c0..8219d32 100644 --- a/openeuler_obs.py +++ b/openeuler_obs.py @@ -113,6 +113,9 @@ par.add_argument("-a", "--ALL_", default=False, help="update all obs repo rpms", par.add_argument("-cmp", "--compare", default=False, help="compare rpm", required=False) par.add_argument("-cpm", "--check_pckg_mgmt", default=False, help="check if there are any problems with the commi for release-management", required=False) +par.add_argument("-ju", "--jenkins_user", help="jekins user name", required=False) +par.add_argument("-jt", "--jenkins_api_token", help="jenkins api token", required=False) +par.add_argument("-jbu", "--jenkins_build_url", help="jenkins build url", required=False) args = par.parse_args() #apply @@ -158,7 +161,10 @@ kw = { "pckg_mgmt": args.pckg_mgmt, "release_management_path": args.remt, "check_pckg_mgmt": args.check_pckg_mgmt, - "compare": args.compare + "compare": args.compare, + "jenkins_user": args.jenkins_user, + "jenkins_api_token": args.jenkins_api_token, + "jenkins_build_url": args.jenkins_build_url } run = Runner(**kw) -- Gitee