From 3f348630a675f9180760f6a360dea3f63547ee4f Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 7 Jun 2022 18:13:34 +0800 Subject: [PATCH 01/10] =?UTF-8?q?1.=E5=90=91obs=5Fmeta=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=85=BC=E5=AE=B9=E6=96=B0=E6=96=B9=E6=A1=88?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E7=BB=93=E6=9E=84=202.=E5=A2=9E=E5=8A=A0mast?= =?UTF-8?q?er/multi-version=E7=9A=84=E5=90=8C=E6=AD=A5=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/sync_pckg_mgmt.py | 299 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 267 insertions(+), 32 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index e0e18d9..a8f571a 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -21,6 +21,7 @@ import os import sys import yaml import shutil +import datetime now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(os.path.join(now_path, "..")) from common.log_obs import log @@ -125,6 +126,18 @@ class SyncPckgMgmt(object): del_msg.append(tmp) return msg, del_msg, prj_pkg + def _parse_yaml_msg_new(self, file_msg): + """ + parse new version yaml file msg + """ + msg = [] + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + msg.append(tmp) + return msg + def _write_prj_meta_file(self, file_path, proj): """ write project meta file @@ -192,8 +205,16 @@ class SyncPckgMgmt(object): """ add obs_meta packages _service file """ - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) pkg_service_path = os.path.join(pkg_path, "_service") if tmp['branch_from'] == "master": branch = "openEuler" @@ -214,28 +235,34 @@ class SyncPckgMgmt(object): def _add_prj_meta_pkgs_service(self, msg): for tmp in msg: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to']: - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'Multi-Version' not in tmp['branch_to']: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to']: + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) self._add_pkg_service(tmp) def _del_pkg(self, tmp): """ delete obs_meta packages """ - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) if os.path.exists(pkg_path): shutil.rmtree(pkg_path) log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] def _verify_meta_file(self, prj_pkg): """ @@ -243,19 +270,26 @@ class SyncPckgMgmt(object): """ for proj, pkg in prj_pkg.items(): if proj != "branch": - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) need_del_pkg = set(meta_pkglist) - set(pkg) log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) for del_pkg in need_del_pkg: tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} self._del_pkg(tmp) - def _push_code(self): + def _push_code(self,repo): """ push code to gitee repo """ - if os.path.exists(self.obs_meta_path): - os.chdir(self.obs_meta_path) + if os.path.exists(repo): + os.chdir(repo) cmd = "git status -s" if os.popen(cmd).read(): cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" @@ -272,34 +306,235 @@ class SyncPckgMgmt(object): log.info("No change, nothing to commit!") return "nothing to push" else: - log.error("%s not exist!" % self.obs_meta_path) + log.error("%s not exist!" % repo) return -1 + def _dispose_msg_packages(self, msg, branch_infos): + ''' + add obs_meta packages _service file from new version yaml file + ''' + change_pkgs = [] + dir_type = branch_infos[1] + if dir_type == 'delete': + for tmp in msg: + pkg_name = self._del_pkg(tmp) + if pkg_name: + change_pkgs.append(pkg_name) + else: + for tmp in msg: + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + return change_pkgs + + def _add_master_pkg_service(self,package): + """ + create and add master service + """ + filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) + if not os.path.exists(filepath): + os.makedirs(filepath) + file_msg = """ + + git + git@gitee.com:src-openeuler/{}.git + * + * + master + + """.format(package) + try: + with open(os.path.join(filepath,'_service'),'w') as f: + f.write(file_msg) + log.info("add %s _service success!" % (filepath)) + except Exception as e: + print (e) + log.info("add %s _service failed!" % (filepath)) + + def _delete_master_pkgs(self,del_pkgs,branch): + ''' + delete pkgs from master branch + ''' + project_name = branch.replace('-',":") + for pkg in del_pkgs: + pkg_path = os.path.join(self.obs_meta_path, 'master', project_name, pkg) + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + log.info("delete master %s %s succeed!" % (project_name,pkg)) + + def _get_master_mainline_packages(self): + ''' + get all mainline packages from now release_management + ''' + branch_lists = os.listdir(os.path.join(self.release_management_path, 'master')) + yaml_dict = {'packages':[]} + for branch in branch_lists: + if branch not in ['openEuler-Epol','openEuler-Factory']: + file_path = os.path.join(self.release_management_path,'master',branch,'pckg-mgmt.yaml') + temp_dict = self._get_yaml_file_msg(file_path) + if temp_dict: + yaml_dict['packages'].extend(temp_dict['packages']) + return yaml_dict + + + def _deal_with_master_packages(self, master_dict): + """ + move or delete obs_meta master packages _service file + """ + change_pkgs = [] + info_msgs = [] + openEuler_Factory = master_dict.get('openEuler-Factory',{'add':[],'del':[]}) + openEuler_Epol = master_dict.get('openEuler-Epol',{'add':[],'del':[]}) + openEuler_Mainline = master_dict.get('openEuler-Mainline',{'add':[],'del':[]}) + for pkg in openEuler_Factory['add']: + self._add_master_pkg_service(pkg) + epol_add_from_fac = list(set(openEuler_Epol['add']).intersection(set(openEuler_Factory['del']))) + info_msgs.append({'to_branch':'openEuler:Epol','from_branch':'openEuler:Factory','pkgs':epol_add_from_fac,'type':'move'}) + mainline_add_from_fac = list(set(openEuler_Mainline['add']).intersection(set(openEuler_Factory['del']))) + info_msgs.append({'to_branch':'openEuler:Mainline','from_branch':'openEuler:Factory','pkgs':mainline_add_from_fac,'type':'move'}) + epol_add_from_main = list(set(openEuler_Epol['add']).intersection(set(openEuler_Mainline['del']))) + info_msgs.append({'to_branch':'openEuler:Epol','from_branch':'openEuler:Mainline','pkgs':epol_add_from_main,'type':'move'}) + main_add_from_epol = list(set(openEuler_Mainline['add']).intersection(set(openEuler_Epol['del']))) + info_msgs.append({'to_branch':'openEuler:Mainline','from_branch':'openEuler:Epol','pkgs':main_add_from_epol,'type':'move'}) + fac_move = epol_add_from_fac + mainline_add_from_fac + real_fac_del = list(set(openEuler_Factory['del']).difference(set(fac_move))) + info_msgs.append({'to_branch':'openEuler:Factory','pkgs':real_fac_del,'type':'delete'}) + real_epol_del = list(set(openEuler_Epol['del']).difference(set(openEuler_Mainline['add']))) + info_msgs.append({'to_branch':'openEuler:Epol','pkgs':real_epol_del,'type':'delete'}) + real_mainline_del = list(set(openEuler_Mainline['del']).difference(set(openEuler_Epol['add']))) + info_msgs.append({'to_branch':'openEuler:Mainline','pkgs':real_mainline_del,'type':'delete'}) + for info in info_msgs: + if info['type'] == 'move': + for pkg in info['pkgs']: + from_pkg_path = os.path.join(self.obs_meta_path, 'master', info['from_branch'], pkg) + pkg_path = os.path.join(self.obs_meta_path, 'master', info['to_branch'], pkg) + mv_to_path = os.path.join(self.obs_meta_path, 'master', info['to_branch']) + if not os.path.exists(pkg_path): + cmd = "mv %s %s" % (from_pkg_path,mv_to_path) + if os.system(cmd) == 0: + change_pkgs.append(pkg) + log.info("move %s from %s to %s _service succeed!" % (pkg,info['from_branch'],info['to_branch'])) + else: + log.info("move %s from %s to %s _service failed!" % (pkg,info['from_branch'],info['to_branch'])) + else: + if info['pkgs']: + change_pkgs += info['pkgs'] + self._delete_master_pkgs(info['pkgs'], info['to_branch']) + return change_pkgs + + def _parse_master_yaml_msg(self,file_msg,branch): + """ + parse master yaml file msg + """ + branch_result = {} + packages = [] + for pckg in file_msg['packages']: + packages.append(pckg['name']) + project_name = branch.replace('-',":") + pkg_meta_path = os.path.join(self.obs_meta_path, 'master', project_name) + meta_packages = os.listdir(pkg_meta_path) + del_packages = list(set(meta_packages).difference(set(packages))) #meta_packages中有而packages中没有的 + add_packages = list(set(packages).difference(set(meta_packages))) + branch_result = {'add':add_packages,'del':del_packages} + return branch_result + + def _write_release_yaml(self,change_pkgs,branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + print (commitid) + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':commitid, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + def sync_yaml_meta(self): """ integration of functions """ change_file = self._get_change_file() + master_change_file = [] + master_dict = {} yaml_dict = {} for line in change_file: log.info("line:%s" % line) name = list(line.split())[1] - branch = name.split('/')[0] file_path = os.path.join(self.release_management_path, name) yaml_dict = self._get_yaml_file_msg(file_path) - if not yaml_dict: - log.info("%s file content is empty!" % name) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = name.split('/')[1] + if branch not in ['openEuler-Epol','openEuler-Factory']: + if master_dict.get('openEuler-Mainline',''): + continue + else: + branch = 'openEuler-Mainline' + yaml_dict = self._get_master_mainline_packages() + master_dict[branch] = self._parse_master_yaml_msg(yaml_dict,branch) else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg) + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + msg = self._parse_yaml_msg_new(yaml_dict) + change_pkgs=self._dispose_msg_packages(msg,branch_infos) + self._write_release_yaml(change_pkgs,branch_infos[0]) else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg) - for tmp in del_msg: - self._del_pkg(tmp) - self._verify_meta_file(prj_pkg) - ret = self._push_code() + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + self._add_prj_meta_pkgs_service(msg) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + self._add_prj_meta_pkgs_service(msg) + for tmp in del_msg: + self._del_pkg(tmp) + self._verify_meta_file(prj_pkg) + if master_dict: + change_pkgs = self._deal_with_master_packages(master_dict) + self._write_release_yaml(change_pkgs,'master') + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) return ret -- Gitee From 4a1931b5f62992f3d84f847022682bfa98a6bb48 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 10 Jun 2022 15:51:57 +0800 Subject: [PATCH 02/10] change commitid to pull request --- core/sync_pckg_mgmt.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index a8f571a..19fdc85 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -18,6 +18,7 @@ Synchronize the obs_meta file according to the pckg-mgmt.yaml file """ import os +import re import sys import yaml import shutil @@ -472,15 +473,21 @@ class SyncPckgMgmt(object): os.chdir(self.release_management_path) commit_cmd = 'git rev-parse HEAD' commitid = os.popen(commit_cmd).read().split('\n')[0] - print (commitid) content_cmd = "git log --oneline -1" content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + else: + pull_request = commitid datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") with open(release_change_yaml) as file: result = yaml.load(file, Loader=yaml.FullLoader) change_dic = { - 'pr':commitid, + 'pr':pull_request, 'description':content, 'changed_packages':change_str, 'date':datestr -- Gitee From 0fd66954d2c8705a92a060f61bc8941d04c58776 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 17 Jun 2022 15:07:39 +0800 Subject: [PATCH 03/10] 0617 --- core/sync_pckg_mgmt.py | 126 +++++++------------------- tools/create_master_pckg_mgmt_yaml.py | 77 ++++++++++++++++ 2 files changed, 111 insertions(+), 92 deletions(-) create mode 100644 tools/create_master_pckg_mgmt_yaml.py diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 19fdc85..1985dcf 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -99,6 +99,14 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) else: for label in file_msg['packages']['everything']: for pckg in file_msg['packages']['everything'][label]: @@ -233,8 +241,22 @@ class SyncPckgMgmt(object): log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.error("copy %s service file failed!" % tmp['pkgname']) - - def _add_prj_meta_pkgs_service(self, msg): + + def _move_master_pkg_service(self, tmp): + """ + move branch master obs_meta packages _service file + """ + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + if not os.path.exists(pkg_path): + cmd = "mv %s %s" % (from_pkg_path,mv_to_path) + if os.system(cmd) == 0: + log.info("move %s from %s to %s _service succeed!" % (pkg,tmp['obs_from'],tmp['obs_to'])) + else: + log.info("move %s from %s to %s _service failed!" % (pkg,tmp['obs_from'],tmp['obs_to'])) + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): for tmp in msg: if 'Multi-Version' not in tmp['branch_to']: prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" @@ -249,7 +271,10 @@ class SyncPckgMgmt(object): if not os.path.exists(selfbuild_meta_path): if "Epol" not in tmp['obs_to']: self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - self._add_pkg_service(tmp) + if 'master' in branch_infos: + self._move_master_pkg_service(tmp) + else: + self._add_pkg_service(tmp) def _del_pkg(self, tmp): """ @@ -388,81 +413,6 @@ class SyncPckgMgmt(object): shutil.rmtree(pkg_path) log.info("delete master %s %s succeed!" % (project_name,pkg)) - def _get_master_mainline_packages(self): - ''' - get all mainline packages from now release_management - ''' - branch_lists = os.listdir(os.path.join(self.release_management_path, 'master')) - yaml_dict = {'packages':[]} - for branch in branch_lists: - if branch not in ['openEuler-Epol','openEuler-Factory']: - file_path = os.path.join(self.release_management_path,'master',branch,'pckg-mgmt.yaml') - temp_dict = self._get_yaml_file_msg(file_path) - if temp_dict: - yaml_dict['packages'].extend(temp_dict['packages']) - return yaml_dict - - - def _deal_with_master_packages(self, master_dict): - """ - move or delete obs_meta master packages _service file - """ - change_pkgs = [] - info_msgs = [] - openEuler_Factory = master_dict.get('openEuler-Factory',{'add':[],'del':[]}) - openEuler_Epol = master_dict.get('openEuler-Epol',{'add':[],'del':[]}) - openEuler_Mainline = master_dict.get('openEuler-Mainline',{'add':[],'del':[]}) - for pkg in openEuler_Factory['add']: - self._add_master_pkg_service(pkg) - epol_add_from_fac = list(set(openEuler_Epol['add']).intersection(set(openEuler_Factory['del']))) - info_msgs.append({'to_branch':'openEuler:Epol','from_branch':'openEuler:Factory','pkgs':epol_add_from_fac,'type':'move'}) - mainline_add_from_fac = list(set(openEuler_Mainline['add']).intersection(set(openEuler_Factory['del']))) - info_msgs.append({'to_branch':'openEuler:Mainline','from_branch':'openEuler:Factory','pkgs':mainline_add_from_fac,'type':'move'}) - epol_add_from_main = list(set(openEuler_Epol['add']).intersection(set(openEuler_Mainline['del']))) - info_msgs.append({'to_branch':'openEuler:Epol','from_branch':'openEuler:Mainline','pkgs':epol_add_from_main,'type':'move'}) - main_add_from_epol = list(set(openEuler_Mainline['add']).intersection(set(openEuler_Epol['del']))) - info_msgs.append({'to_branch':'openEuler:Mainline','from_branch':'openEuler:Epol','pkgs':main_add_from_epol,'type':'move'}) - fac_move = epol_add_from_fac + mainline_add_from_fac - real_fac_del = list(set(openEuler_Factory['del']).difference(set(fac_move))) - info_msgs.append({'to_branch':'openEuler:Factory','pkgs':real_fac_del,'type':'delete'}) - real_epol_del = list(set(openEuler_Epol['del']).difference(set(openEuler_Mainline['add']))) - info_msgs.append({'to_branch':'openEuler:Epol','pkgs':real_epol_del,'type':'delete'}) - real_mainline_del = list(set(openEuler_Mainline['del']).difference(set(openEuler_Epol['add']))) - info_msgs.append({'to_branch':'openEuler:Mainline','pkgs':real_mainline_del,'type':'delete'}) - for info in info_msgs: - if info['type'] == 'move': - for pkg in info['pkgs']: - from_pkg_path = os.path.join(self.obs_meta_path, 'master', info['from_branch'], pkg) - pkg_path = os.path.join(self.obs_meta_path, 'master', info['to_branch'], pkg) - mv_to_path = os.path.join(self.obs_meta_path, 'master', info['to_branch']) - if not os.path.exists(pkg_path): - cmd = "mv %s %s" % (from_pkg_path,mv_to_path) - if os.system(cmd) == 0: - change_pkgs.append(pkg) - log.info("move %s from %s to %s _service succeed!" % (pkg,info['from_branch'],info['to_branch'])) - else: - log.info("move %s from %s to %s _service failed!" % (pkg,info['from_branch'],info['to_branch'])) - else: - if info['pkgs']: - change_pkgs += info['pkgs'] - self._delete_master_pkgs(info['pkgs'], info['to_branch']) - return change_pkgs - - def _parse_master_yaml_msg(self,file_msg,branch): - """ - parse master yaml file msg - """ - branch_result = {} - packages = [] - for pckg in file_msg['packages']: - packages.append(pckg['name']) - project_name = branch.replace('-',":") - pkg_meta_path = os.path.join(self.obs_meta_path, 'master', project_name) - meta_packages = os.listdir(pkg_meta_path) - del_packages = list(set(meta_packages).difference(set(packages))) #meta_packages中有而packages中没有的 - add_packages = list(set(packages).difference(set(meta_packages))) - branch_result = {'add':add_packages,'del':del_packages} - return branch_result def _write_release_yaml(self,change_pkgs,branch): ''' @@ -503,7 +453,6 @@ class SyncPckgMgmt(object): """ change_file = self._get_change_file() master_change_file = [] - master_dict = {} yaml_dict = {} for line in change_file: log.info("line:%s" % line) @@ -513,13 +462,9 @@ class SyncPckgMgmt(object): branch_infos = name.split('/') if 'master' in branch_infos: branch = name.split('/')[1] - if branch not in ['openEuler-Epol','openEuler-Factory']: - if master_dict.get('openEuler-Mainline',''): - continue - else: - branch = 'openEuler-Mainline' - yaml_dict = self._get_master_mainline_packages() - master_dict[branch] = self._parse_master_yaml_msg(yaml_dict,branch) + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + self._add_prj_meta_pkgs_service(msg,branch_infos) + self._verify_meta_file(prj_pkg) else: if not yaml_dict: log.info("%s file content is empty!" % name) @@ -530,18 +475,15 @@ class SyncPckgMgmt(object): else: if "everything" in yaml_dict['packages'].keys(): msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg) + self._add_prj_meta_pkgs_service(msg, branch_infos) else: msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg) + self._add_prj_meta_pkgs_service(msg, branch_infos) for tmp in del_msg: self._del_pkg(tmp) self._verify_meta_file(prj_pkg) - if master_dict: - change_pkgs = self._deal_with_master_packages(master_dict) - self._write_release_yaml(change_pkgs,'master') ret = self._push_code(self.obs_meta_path) - self._push_code(self.release_management_path) + # self._push_code(self.release_management_path) return ret diff --git a/tools/create_master_pckg_mgmt_yaml.py b/tools/create_master_pckg_mgmt_yaml.py new file mode 100644 index 0000000..477722e --- /dev/null +++ b/tools/create_master_pckg_mgmt_yaml.py @@ -0,0 +1,77 @@ +#/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: dongjie +# Create: 2022-06-16 +# ****************************************************************************** +""" +create pckg-mgmt.yaml +""" + +import argparse +import os +import sys +import yaml +import datetime +import pandas as pd + +# all_dict = {"openEuler-Factory":['A','B'],"openEuler-MultiLanguage":['C','D']} +# pkgs_dict = {"packages": {"everything": {"baseos": [], "other": []},"epol": [], "recycle": [], "delete": []}} +datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + +def write_yaml(dict_msg, file_path): + with open(file_path, "w", encoding='utf-8') as f: + yaml.dump(dict_msg, f, default_flow_style=False, sort_keys=False) + +def parse_pkgs(all_dict): + for branch,pkgs in all_dict.items(): + print (branch) + pkgs_dict = {"packages":[]} + yaml_path = os.path.join(os.getcwd(),"{}/pckg-mgmt.yaml".format(branch)) + dir_path = os.path.join(os.getcwd(),branch) + if not os.path.isdir(dir_path): + os.makedirs(dir_path) + # pkgs_dict['packages'] = pkgs + if branch in ['openEuler-Factory','openEuler-Epol']: + obs_from = branch.replace("-",":") + obs_to = "{}:Testing:{}".format(branch.split("-")[0],branch.split("-")[1]) + else: + obs_from = 'openEuler:Mainline' + obs_to = "{}:Testing:{}".format(branch.split("-")[0],branch.split("-")[1]) + for pkg in pkgs: + pkgs_dict["packages"].append({ + "name": pkg, + # "branch_from": from_branch, + # "branch_to": to_branch, + "obs_from": obs_from, + "obs_to": obs_to, + "date": datestr + }) + write_yaml(pkgs_dict,yaml_path) + +def parse_excel(excel_path): + all_dict = {} + df = pd.read_excel(excel_path,sheet_name=1,header =0) + for index,row in df.iterrows(): + project = row[1].replace(":","-") + pkg = row[0] + if all_dict.get(project,''): + all_dict[project].append(pkg) + else: + all_dict[project] = [pkg] + return all_dict + + +if __name__ == '__main__': + excel_path = r'D:\分层构建\layer.xlsx' + all_dict = parse_excel(excel_path) + parse_pkgs(all_dict) \ No newline at end of file -- Gitee From 150f730ec5e26040d0ab98b1a12fa8e974a24c4e Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Mon, 20 Jun 2022 10:26:31 +0800 Subject: [PATCH 04/10] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E5=88=86=E5=B1=82=E6=9E=B6=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/sync_pckg_mgmt.py | 144 +++++++++++++++++++---------------------- 1 file changed, 67 insertions(+), 77 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 1985dcf..f5a6966 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -123,16 +123,17 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) + if yaml_type != 'master': + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) return msg, del_msg, prj_pkg def _parse_yaml_msg_new(self, file_msg): @@ -242,19 +243,39 @@ class SyncPckgMgmt(object): else: log.error("copy %s service file failed!" % tmp['pkgname']) - def _move_master_pkg_service(self, tmp): + def _move_master_pkg_service(self, msg): """ move branch master obs_meta packages _service file """ - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) - if not os.path.exists(pkg_path): - cmd = "mv %s %s" % (from_pkg_path,mv_to_path) - if os.system(cmd) == 0: - log.info("move %s from %s to %s _service succeed!" % (pkg,tmp['obs_from'],tmp['obs_to'])) - else: - log.info("move %s from %s to %s _service failed!" % (pkg,tmp['obs_from'],tmp['obs_to'])) + # ****************************************** + # from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + # pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + # mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + # if not os.path.exists(mv_to_path): + # os.makedirs(mv_to_path) + # if not os.path.exists(pkg_path): + # cmd = "mv %s %s" % (from_pkg_path,mv_to_path) + # if os.system(cmd) == 0: + # log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + # else: + # log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + # ******************************************** + change_pkgs = [] + for tmp in msg: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return change_pkgs + def _add_prj_meta_pkgs_service(self, msg, branch_infos): for tmp in msg: @@ -271,9 +292,7 @@ class SyncPckgMgmt(object): if not os.path.exists(selfbuild_meta_path): if "Epol" not in tmp['obs_to']: self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - if 'master' in branch_infos: - self._move_master_pkg_service(tmp) - else: + if 'master' not in branch_infos: self._add_pkg_service(tmp) def _del_pkg(self, tmp): @@ -309,6 +328,7 @@ class SyncPckgMgmt(object): for del_pkg in need_del_pkg: tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} self._del_pkg(tmp) + return list(need_del_pkg) def _push_code(self,repo): """ @@ -378,42 +398,6 @@ class SyncPckgMgmt(object): log.error("copy %s service file failed!" % tmp['pkgname']) return change_pkgs - def _add_master_pkg_service(self,package): - """ - create and add master service - """ - filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) - if not os.path.exists(filepath): - os.makedirs(filepath) - file_msg = """ - - git - git@gitee.com:src-openeuler/{}.git - * - * - master - - """.format(package) - try: - with open(os.path.join(filepath,'_service'),'w') as f: - f.write(file_msg) - log.info("add %s _service success!" % (filepath)) - except Exception as e: - print (e) - log.info("add %s _service failed!" % (filepath)) - - def _delete_master_pkgs(self,del_pkgs,branch): - ''' - delete pkgs from master branch - ''' - project_name = branch.replace('-',":") - for pkg in del_pkgs: - pkg_path = os.path.join(self.obs_meta_path, 'master', project_name, pkg) - if os.path.exists(pkg_path): - shutil.rmtree(pkg_path) - log.info("delete master %s %s succeed!" % (project_name,pkg)) - - def _write_release_yaml(self,change_pkgs,branch): ''' write change info to release_change.yaml @@ -430,22 +414,24 @@ class SyncPckgMgmt(object): if match: pr_id = match.group(0) pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") else: pull_request = commitid - datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') - release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") - with open(release_change_yaml) as file: - result = yaml.load(file, Loader=yaml.FullLoader) - change_dic = { - 'pr':pull_request, - 'description':content, - 'changed_packages':change_str, - 'date':datestr - } - result['release-change'].append(change_dic) - with open(release_change_yaml, "w", encoding='utf-8') as f: - yaml.dump(result, f, default_flow_style=False, sort_keys=False) - log.info("write release change yaml file success") + log.info("ignore write release change yaml file success") + def sync_yaml_meta(self): """ @@ -464,7 +450,10 @@ class SyncPckgMgmt(object): branch = name.split('/')[1] msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") self._add_prj_meta_pkgs_service(msg,branch_infos) - self._verify_meta_file(prj_pkg) + move_change_pkgs = self._move_master_pkg_service(msg) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = move_change_pkgs + del_change_pkgs + self._write_release_yaml(change_pkgs,branch_infos[0]) else: if not yaml_dict: log.info("%s file content is empty!" % name) @@ -481,13 +470,14 @@ class SyncPckgMgmt(object): self._add_prj_meta_pkgs_service(msg, branch_infos) for tmp in del_msg: self._del_pkg(tmp) - self._verify_meta_file(prj_pkg) + del_change_pkgs = self._verify_meta_file(prj_pkg) ret = self._push_code(self.obs_meta_path) - # self._push_code(self.release_management_path) + self._push_code(self.release_management_path) return ret if __name__ == "__main__": - kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} mgmt = SyncPckgMgmt(**kw) mgmt.sync_yaml_meta() -- Gitee From 4409e7313e3626821cd4a3e2b998ab121335cd00 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Mon, 20 Jun 2022 18:24:19 +0800 Subject: [PATCH 05/10] wq --- config/config.ini | 19 +++++++++++++++++-- core/sync_pckg_mgmt.py | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/config/config.ini b/config/config.ini index 5ae97f3..f00c9ea 100644 --- a/config/config.ini +++ b/config/config.ini @@ -30,7 +30,7 @@ name = ci_check build [package_info_file] name = package_info.csv [branch_proj] -master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras +master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras openEuler:Testing:BaseTools openEuler:Testing:C openEuler:Testing:Common_Languages_Dependent_Tools openEuler:Testing:Epol openEuler:Testing:Erlang openEuler:Testing:Factory openEuler:Testing:Java openEuler:Testing:KernelSpace openEuler:Testing:Mainline openEuler:Testing:Meson openEuler:Testing:MultiLanguage openEuler:Testing:Nodejs openEuler:Testing:Perl openEuler:Testing:Python openEuler:Testing:Ruby openEuler-20.03-LTS = openEuler:20.03:LTS openEuler:20.03:LTS:Epol openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1 openEuler:20.03:LTS:SP1:Epol openEuler:20.03:LTS:SP1:Extras openEuler-20.03-LTS-Next = openEuler:20.03:LTS:Next openEuler:20.03:LTS:Next:Epol openEuler:20.03:LTS:Next:Extras @@ -65,6 +65,21 @@ obs_pkg_rpms = https://gitee.com/unsunghero/obs_pkg_rpms openEuler-Mainline = openEuler:selfbuild:BaseOS/mainline_standard_aarch64 openEuler:selfbuild:BaseOS/mainline_standard_x86_64 openEuler-Epol = openEuler:selfbuild:BaseOS/epol_aarch64 openEuler:selfbuild:BaseOS/epol_x86_64 openEuler-Factory = openEuler:selfbuild:BaseOS/factory_aarch64 openEuler:selfbuild:BaseOS/factory_x86_64 +openEuler-Testing-BaseTools = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-C = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Common_Languages_Dependent_Tools = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Epol = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Erlang = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Factory = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Java = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-KernelSpace = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Mainline = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Meson = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-MultiLanguage = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Nodejs = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Perl = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Python = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Ruby = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-20.03-LTS = openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_aarch64 openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_x86_64 openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_x86_64 openEuler-20.03-LTS-SP1-Epol = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_x86_64 @@ -105,4 +120,4 @@ path = /srv/obs/build [obs_ignore_package] name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv [obs_include_project] -name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely \ No newline at end of file +name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:Testing:BaseTools openEuler:Testing:C openEuler:Testing:Common_Languages_Dependent_Tools openEuler:Testing:Epol openEuler:Testing:Erlang openEuler:Testing:Factory openEuler:Testing:Java openEuler:Testing:KernelSpace openEuler:Testing:Mainline openEuler:Testing:Meson openEuler:Testing:MultiLanguage openEuler:Testing:Nodejs openEuler:Testing:Perl openEuler:Testing:Python openEuler:Testing:Ruby \ No newline at end of file diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index f5a6966..53b7314 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -290,7 +290,7 @@ class SyncPckgMgmt(object): selfbuild_meta_path = os.path.join(prj_meta_br, "%s:selfbuild:BaseOS" % tmp['obs_to']) if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to']: + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) if 'master' not in branch_infos: self._add_pkg_service(tmp) -- Gitee From e7cd01f641bc3a1bc5c2e0d3c0ee7c1ab91410c7 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 21 Jun 2022 10:39:02 +0800 Subject: [PATCH 06/10] ii --- core/sync_pckg_mgmt.py | 985 +++++++++++++++++++++-------------------- 1 file changed, 502 insertions(+), 483 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 53b7314..54f9c87 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -1,483 +1,502 @@ -#!/bin/env python3 -# -*- encoding=utf8 -*- -#****************************************************************************** -# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. -# licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Author: wangchong -# Create: 2021-06-08 -# ****************************************************************************** -""" -Synchronize the obs_meta file according to the pckg-mgmt.yaml file -""" - -import os -import re -import sys -import yaml -import shutil -import datetime -now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) -sys.path.append(os.path.join(now_path, "..")) -from common.log_obs import log - - -class SyncPckgMgmt(object): - """ - keep pckg-mgmt.yaml and obs_meta in sync - """ - def __init__(self, **kwargs): - """ - giteeuser: gitee user name - giteeuserpwd: gitee password - """ - self.kwargs = kwargs - self.giteeuser = self.kwargs['gitee_user'] - self.giteeuserpwd = self.kwargs['gitee_pwd'] - self.obs_meta_path = self.kwargs['obs_meta_path'] - self.release_management_path = self.kwargs['release_management_path'] - - def _get_change_file(self): - """ - get release-managemnet change file - """ - if os.path.exists(self.release_management_path): - os.chdir(self.release_management_path) - cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" - result = os.popen(cmd).read().split('\n') - change_file = [x for x in result if x != ''] - return change_file - else: - log.error("%s not exist!" % self.release_management_path) - sys.exit(1) - - def _get_yaml_file_msg(self, file_path): - """ - get pckg-mgmt.yaml file msg - """ - file_msg = None - if os.path.exists(file_path): - with open(file_path, "r", encoding='utf-8') as f: - file_msg = yaml.load(f, Loader=yaml.FullLoader) - return file_msg - - def _check_pckg_yaml(self, file_msg, branch): - """ - check pckg-mgmt.yaml file - """ - proj = branch.replace("-", ":") - flag = False - for title in file_msg['packages']: - for msg in file_msg['packages'][title]: - if msg['branch_to'] != branch: - flag = True - log.error("%s branch_to is error, please check yaml." % msg['name']) - if not msg['obs_to'].startswith(proj): - flag = True - log.error("%s obs_to is error, please check yaml." % msg['name']) - return flag - - def _parse_yaml_msg(self, file_msg, yaml_type): - """ - parse yaml file msg - """ - tmp = {} - msg = [] - del_msg = [] - prj_pkg = {} - if yaml_type == "old": - for pckg in file_msg['packages']['natural']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - elif yaml_type == 'master': - for pckg in file_msg['packages']: - tmp = {'pkgname': pckg['name'], 'branch_from': 'master', - 'branch_to': 'master', 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = 'master' - msg.append(tmp) - else: - for label in file_msg['packages']['everything']: - for pckg in file_msg['packages']['everything'][label]: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - for pckg in file_msg['packages']['epol']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - if yaml_type != 'master': - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) - return msg, del_msg, prj_pkg - - def _parse_yaml_msg_new(self, file_msg): - """ - parse new version yaml file msg - """ - msg = [] - for pckg in file_msg['packages']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - msg.append(tmp) - return msg - - def _write_prj_meta_file(self, file_path, proj): - """ - write project meta file - """ - if "Epol" in proj: - main_proj = proj.replace(':Epol', '') - con_proj = main_proj.replace(':', '_').lower() - epol_repo_aarch64 = "\n " % ( - main_proj, con_proj) - epol_repo_x86 = "\n " % ( - main_proj, con_proj) - else: - main_proj = proj - con_proj = main_proj.replace(':', '_').lower() - epol_repo_aarch64 = "" - epol_repo_x86 = "" - file_msg = """ - - <description/> - <person userid="Admin" role="maintainer"/> - <useforbuild> - <disable/> - </useforbuild> - <repository name="standard_aarch64"> - <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_aarch64"/>{3} - <arch>aarch64</arch> - </repository> - <repository name="standard_x86_64"> - <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_x86_64"/>{4} - <arch>x86_64</arch> - </repository> -</project> -""".format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) - f = open(file_path, "w") - f.write(file_msg) - f.close() - - def _write_selfbuild_meta_file(self, file_path, proj): - """ - write selfbuild project meta file - """ - file_msg = """<project name="{0}:selfbuild:BaseOS"> - <title/> - <description/> - <person userid="Admin" role="maintainer"/> - <repository name="{1}_standard_aarch64"> - <arch>aarch64</arch> - </repository> - <repository name="{1}_standard_x86_64"> - <arch>x86_64</arch> - </repository> - <repository name="{1}_epol_aarch64"> - <arch>aarch64</arch> - </repository> - <repository name="{1}_epol_x86_64"> - <arch>x86_64</arch> - </repository> -</project> -""".format(proj, proj.lower().replace(':', '_')) - f = open(file_path, "w") - f.write(file_msg) - f.close() - - def _add_pkg_service(self, tmp): - """ - add obs_meta packages _service file - """ - if 'Multi-Version' in tmp['branch_from']: - dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) - from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) - pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if tmp['branch_from'] == "master": - branch = "openEuler" - else: - branch = tmp['branch_from'] - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) - if os.system(cmd) == 0: - log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.error("copy %s service file failed!" % tmp['pkgname']) - - def _move_master_pkg_service(self, msg): - """ - move branch master obs_meta packages _service file - """ - # ****************************************** - # from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - # pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - # mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) - # if not os.path.exists(mv_to_path): - # os.makedirs(mv_to_path) - # if not os.path.exists(pkg_path): - # cmd = "mv %s %s" % (from_pkg_path,mv_to_path) - # if os.system(cmd) == 0: - # log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - # else: - # log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - # ******************************************** - change_pkgs = [] - for tmp in msg: - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - change_pkgs.append(tmp['pkgname']) - log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - else: - log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - return change_pkgs - - - def _add_prj_meta_pkgs_service(self, msg, branch_infos): - for tmp in msg: - if 'Multi-Version' not in tmp['branch_to']: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - if 'master' not in branch_infos: - self._add_pkg_service(tmp) - - def _del_pkg(self, tmp): - """ - delete obs_meta packages - """ - if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) - pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - if os.path.exists(pkg_path): - shutil.rmtree(pkg_path) - log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - return tmp['pkgname'] - - def _verify_meta_file(self, prj_pkg): - """ - verify obs_meta with pckg-mgmt.yaml - """ - for proj, pkg in prj_pkg.items(): - if proj != "branch": - if 'Multi-Version' in prj_pkg['branch']: - dir_name = '{}/{}'.format(prj_pkg['branch'],proj) - meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) - meta_pkglist = os.listdir(meta_path) - if 'README.md' in meta_pkglist: - meta_pkglist.remove('README.md') - else: - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) - need_del_pkg = set(meta_pkglist) - set(pkg) - log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) - for del_pkg in need_del_pkg: - tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} - self._del_pkg(tmp) - return list(need_del_pkg) - - def _push_code(self,repo): - """ - push code to gitee repo - """ - if os.path.exists(repo): - os.chdir(repo) - cmd = "git status -s" - if os.popen(cmd).read(): - cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" - if os.system(cmd) == 0: - cmd = "git push -f" - for i in range(5): - if os.system(cmd) == 0: - log.info("push code to gitee repo succeed!") - return 0 - else: - log.error("push code failed, try again...") - raise SystemExit("push code to gitee repo Failed!") - else: - log.info("No change, nothing to commit!") - return "nothing to push" - else: - log.error("%s not exist!" % repo) - return -1 - - def _dispose_msg_packages(self, msg, branch_infos): - ''' - add obs_meta packages _service file from new version yaml file - ''' - change_pkgs = [] - dir_type = branch_infos[1] - if dir_type == 'delete': - for tmp in msg: - pkg_name = self._del_pkg(tmp) - if pkg_name: - change_pkgs.append(pkg_name) - else: - for tmp in msg: - if 'Multi-Version' in tmp['branch_from']: - dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) - from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) - pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if tmp['branch_from'] == "master": - branch = "openEuler" - else: - branch = tmp['branch_from'] - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) - if os.system(cmd) == 0: - change_pkgs.append(tmp['pkgname']) - log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.error("copy %s service file failed!" % tmp['pkgname']) - return change_pkgs - - def _write_release_yaml(self,change_pkgs,branch): - ''' - write change info to release_change.yaml - ''' - if change_pkgs: - change_str = " ".join(change_pkgs) - os.chdir(self.release_management_path) - commit_cmd = 'git rev-parse HEAD' - commitid = os.popen(commit_cmd).read().split('\n')[0] - content_cmd = "git log --oneline -1" - content = os.popen(content_cmd).read().split('\n')[0] - reg=re.compile(r"(?<=!)\d+") - match=reg.search(content) - if match: - pr_id = match.group(0) - pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) - datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') - release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") - with open(release_change_yaml) as file: - result = yaml.load(file, Loader=yaml.FullLoader) - change_dic = { - 'pr':pull_request, - 'description':content, - 'changed_packages':change_str, - 'date':datestr - } - result['release-change'].append(change_dic) - with open(release_change_yaml, "w", encoding='utf-8') as f: - yaml.dump(result, f, default_flow_style=False, sort_keys=False) - log.info("write release change yaml file success") - else: - pull_request = commitid - log.info("ignore write release change yaml file success") - - - def sync_yaml_meta(self): - """ - integration of functions - """ - change_file = self._get_change_file() - master_change_file = [] - yaml_dict = {} - for line in change_file: - log.info("line:%s" % line) - name = list(line.split())[1] - file_path = os.path.join(self.release_management_path, name) - yaml_dict = self._get_yaml_file_msg(file_path) - branch_infos = name.split('/') - if 'master' in branch_infos: - branch = name.split('/')[1] - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") - self._add_prj_meta_pkgs_service(msg,branch_infos) - move_change_pkgs = self._move_master_pkg_service(msg) - del_change_pkgs = self._verify_meta_file(prj_pkg) - change_pkgs = move_change_pkgs + del_change_pkgs - self._write_release_yaml(change_pkgs,branch_infos[0]) - else: - if not yaml_dict: - log.info("%s file content is empty!" % name) - elif isinstance(yaml_dict['packages'], list): - msg = self._parse_yaml_msg_new(yaml_dict) - change_pkgs=self._dispose_msg_packages(msg,branch_infos) - self._write_release_yaml(change_pkgs,branch_infos[0]) - else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg, branch_infos) - else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg, branch_infos) - for tmp in del_msg: - self._del_pkg(tmp) - del_change_pkgs = self._verify_meta_file(prj_pkg) - ret = self._push_code(self.obs_meta_path) - self._push_code(self.release_management_path) - return ret - - -if __name__ == "__main__": - # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} - kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} - mgmt = SyncPckgMgmt(**kw) - mgmt.sync_yaml_meta() +#!/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: wangchong +# Create: 2021-06-08 +# ****************************************************************************** +""" +Synchronize the obs_meta file according to the pckg-mgmt.yaml file +""" + +import os +import re +import sys +import yaml +import shutil +import datetime +now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) +sys.path.append(os.path.join(now_path, "..")) +from common.log_obs import log + + +class SyncPckgMgmt(object): + """ + keep pckg-mgmt.yaml and obs_meta in sync + """ + def __init__(self, **kwargs): + """ + giteeuser: gitee user name + giteeuserpwd: gitee password + """ + self.kwargs = kwargs + self.giteeuser = self.kwargs['gitee_user'] + self.giteeuserpwd = self.kwargs['gitee_pwd'] + self.obs_meta_path = self.kwargs['obs_meta_path'] + self.release_management_path = self.kwargs['release_management_path'] + + def _get_change_file(self): + """ + get release-managemnet change file + """ + if os.path.exists(self.release_management_path): + os.chdir(self.release_management_path) + cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" + result = os.popen(cmd).read().split('\n') + change_file = [x for x in result if x != ''] + return change_file + else: + log.error("%s not exist!" % self.release_management_path) + sys.exit(1) + + def _get_yaml_file_msg(self, file_path): + """ + get pckg-mgmt.yaml file msg + """ + file_msg = None + if os.path.exists(file_path): + with open(file_path, "r", encoding='utf-8') as f: + file_msg = yaml.load(f, Loader=yaml.FullLoader) + return file_msg + + def _check_pckg_yaml(self, file_msg, branch): + """ + check pckg-mgmt.yaml file + """ + proj = branch.replace("-", ":") + flag = False + for title in file_msg['packages']: + for msg in file_msg['packages'][title]: + if msg['branch_to'] != branch: + flag = True + log.error("%s branch_to is error, please check yaml." % msg['name']) + if not msg['obs_to'].startswith(proj): + flag = True + log.error("%s obs_to is error, please check yaml." % msg['name']) + return flag + + def _parse_yaml_msg(self, file_msg, yaml_type): + """ + parse yaml file msg + """ + tmp = {} + msg = [] + del_msg = [] + prj_pkg = {} + if yaml_type == "old": + for pckg in file_msg['packages']['natural']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) + else: + for label in file_msg['packages']['everything']: + for pckg in file_msg['packages']['everything'][label]: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + for pckg in file_msg['packages']['epol']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + if yaml_type != 'master': + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + return msg, del_msg, prj_pkg + + def _parse_yaml_msg_new(self, file_msg): + """ + parse new version yaml file msg + """ + msg = [] + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + msg.append(tmp) + return msg + + def _write_prj_meta_file(self, file_path, proj): + """ + write project meta file + """ + if "Epol" in proj: + main_proj = proj.replace(':Epol', '') + con_proj = main_proj.replace(':', '_').lower() + epol_repo_aarch64 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_aarch64\"/>" % ( + main_proj, con_proj) + epol_repo_x86 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_x86_64\"/>" % ( + main_proj, con_proj) + else: + main_proj = proj + con_proj = main_proj.replace(':', '_').lower() + epol_repo_aarch64 = "" + epol_repo_x86 = "" + if 'openEuler:Testing' in proj: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <useforbuild> + <disable/> + </useforbuild> + <repository name="standard_aarch64"> + <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64"> + <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <useforbuild> + <disable/> + </useforbuild> + <repository name="standard_aarch64"> + <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_aarch64"/>{3} + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64"> + <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_x86_64"/>{4} + <arch>x86_64</arch> + </repository> +</project> + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + f = open(file_path, "w") + f.write(file_msg) + f.close() + + def _write_selfbuild_meta_file(self, file_path, proj): + """ + write selfbuild project meta file + """ + file_msg = """<project name="{0}:selfbuild:BaseOS"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="{1}_standard_aarch64"> + <arch>aarch64</arch> + </repository> + <repository name="{1}_standard_x86_64"> + <arch>x86_64</arch> + </repository> + <repository name="{1}_epol_aarch64"> + <arch>aarch64</arch> + </repository> + <repository name="{1}_epol_x86_64"> + <arch>x86_64</arch> + </repository> +</project> +""".format(proj, proj.lower().replace(':', '_')) + f = open(file_path, "w") + f.write(file_msg) + f.close() + + def _add_pkg_service(self, tmp): + """ + add obs_meta packages _service file + """ + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + + def _move_master_pkg_service(self, msg): + """ + move branch master obs_meta packages _service file + """ + # ****************************************** + # from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + # pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + # mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + # if not os.path.exists(mv_to_path): + # os.makedirs(mv_to_path) + # if not os.path.exists(pkg_path): + # cmd = "mv %s %s" % (from_pkg_path,mv_to_path) + # if os.system(cmd) == 0: + # log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + # else: + # log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + # ******************************************** + change_pkgs = [] + for tmp in msg: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return change_pkgs + + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): + for tmp in msg: + if 'Multi-Version' not in tmp['branch_to']: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'master' not in branch_infos: + self._add_pkg_service(tmp) + + def _del_pkg(self, tmp): + """ + delete obs_meta packages + """ + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] + + def _verify_meta_file(self, prj_pkg): + """ + verify obs_meta with pckg-mgmt.yaml + """ + for proj, pkg in prj_pkg.items(): + if proj != "branch": + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + need_del_pkg = set(meta_pkglist) - set(pkg) + log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) + for del_pkg in need_del_pkg: + tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} + self._del_pkg(tmp) + return list(need_del_pkg) + + def _push_code(self,repo): + """ + push code to gitee repo + """ + if os.path.exists(repo): + os.chdir(repo) + cmd = "git status -s" + if os.popen(cmd).read(): + cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" + if os.system(cmd) == 0: + cmd = "git push -f" + for i in range(5): + if os.system(cmd) == 0: + log.info("push code to gitee repo succeed!") + return 0 + else: + log.error("push code failed, try again...") + raise SystemExit("push code to gitee repo Failed!") + else: + log.info("No change, nothing to commit!") + return "nothing to push" + else: + log.error("%s not exist!" % repo) + return -1 + + def _dispose_msg_packages(self, msg, branch_infos): + ''' + add obs_meta packages _service file from new version yaml file + ''' + change_pkgs = [] + dir_type = branch_infos[1] + if dir_type == 'delete': + for tmp in msg: + pkg_name = self._del_pkg(tmp) + if pkg_name: + change_pkgs.append(pkg_name) + else: + for tmp in msg: + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + return change_pkgs + + def _write_release_yaml(self,change_pkgs,branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + else: + pull_request = commitid + log.info("ignore write release change yaml file success") + + + def sync_yaml_meta(self): + """ + integration of functions + """ + change_file = self._get_change_file() + master_change_file = [] + yaml_dict = {} + for line in change_file: + log.info("line:%s" % line) + name = list(line.split())[1] + file_path = os.path.join(self.release_management_path, name) + yaml_dict = self._get_yaml_file_msg(file_path) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = name.split('/')[1] + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + self._add_prj_meta_pkgs_service(msg,branch_infos) + move_change_pkgs = self._move_master_pkg_service(msg) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = move_change_pkgs + del_change_pkgs + self._write_release_yaml(change_pkgs,branch_infos[0]) + else: + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + msg = self._parse_yaml_msg_new(yaml_dict) + change_pkgs=self._dispose_msg_packages(msg,branch_infos) + self._write_release_yaml(change_pkgs,branch_infos[0]) + else: + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + self._add_prj_meta_pkgs_service(msg, branch_infos) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + self._add_prj_meta_pkgs_service(msg, branch_infos) + for tmp in del_msg: + self._del_pkg(tmp) + del_change_pkgs = self._verify_meta_file(prj_pkg) + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) + return ret + + +if __name__ == "__main__": + # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} + mgmt = SyncPckgMgmt(**kw) + mgmt.sync_yaml_meta() -- Gitee From c334ff7686203919e71bfb05670e851ea0d326fa Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 21 Jun 2022 10:41:59 +0800 Subject: [PATCH 07/10] dsw --- core/sync_pckg_mgmt.py | 1004 ++++++++++++++++++++-------------------- 1 file changed, 502 insertions(+), 502 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 54f9c87..3060070 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -1,502 +1,502 @@ -#!/bin/env python3 -# -*- encoding=utf8 -*- -#****************************************************************************** -# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. -# licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Author: wangchong -# Create: 2021-06-08 -# ****************************************************************************** -""" -Synchronize the obs_meta file according to the pckg-mgmt.yaml file -""" - -import os -import re -import sys -import yaml -import shutil -import datetime -now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) -sys.path.append(os.path.join(now_path, "..")) -from common.log_obs import log - - -class SyncPckgMgmt(object): - """ - keep pckg-mgmt.yaml and obs_meta in sync - """ - def __init__(self, **kwargs): - """ - giteeuser: gitee user name - giteeuserpwd: gitee password - """ - self.kwargs = kwargs - self.giteeuser = self.kwargs['gitee_user'] - self.giteeuserpwd = self.kwargs['gitee_pwd'] - self.obs_meta_path = self.kwargs['obs_meta_path'] - self.release_management_path = self.kwargs['release_management_path'] - - def _get_change_file(self): - """ - get release-managemnet change file - """ - if os.path.exists(self.release_management_path): - os.chdir(self.release_management_path) - cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" - result = os.popen(cmd).read().split('\n') - change_file = [x for x in result if x != ''] - return change_file - else: - log.error("%s not exist!" % self.release_management_path) - sys.exit(1) - - def _get_yaml_file_msg(self, file_path): - """ - get pckg-mgmt.yaml file msg - """ - file_msg = None - if os.path.exists(file_path): - with open(file_path, "r", encoding='utf-8') as f: - file_msg = yaml.load(f, Loader=yaml.FullLoader) - return file_msg - - def _check_pckg_yaml(self, file_msg, branch): - """ - check pckg-mgmt.yaml file - """ - proj = branch.replace("-", ":") - flag = False - for title in file_msg['packages']: - for msg in file_msg['packages'][title]: - if msg['branch_to'] != branch: - flag = True - log.error("%s branch_to is error, please check yaml." % msg['name']) - if not msg['obs_to'].startswith(proj): - flag = True - log.error("%s obs_to is error, please check yaml." % msg['name']) - return flag - - def _parse_yaml_msg(self, file_msg, yaml_type): - """ - parse yaml file msg - """ - tmp = {} - msg = [] - del_msg = [] - prj_pkg = {} - if yaml_type == "old": - for pckg in file_msg['packages']['natural']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - elif yaml_type == 'master': - for pckg in file_msg['packages']: - tmp = {'pkgname': pckg['name'], 'branch_from': 'master', - 'branch_to': 'master', 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = 'master' - msg.append(tmp) - else: - for label in file_msg['packages']['everything']: - for pckg in file_msg['packages']['everything'][label]: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - for pckg in file_msg['packages']['epol']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - if yaml_type != 'master': - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) - return msg, del_msg, prj_pkg - - def _parse_yaml_msg_new(self, file_msg): - """ - parse new version yaml file msg - """ - msg = [] - for pckg in file_msg['packages']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - msg.append(tmp) - return msg - - def _write_prj_meta_file(self, file_path, proj): - """ - write project meta file - """ - if "Epol" in proj: - main_proj = proj.replace(':Epol', '') - con_proj = main_proj.replace(':', '_').lower() - epol_repo_aarch64 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_aarch64\"/>" % ( - main_proj, con_proj) - epol_repo_x86 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_x86_64\"/>" % ( - main_proj, con_proj) - else: - main_proj = proj - con_proj = main_proj.replace(':', '_').lower() - epol_repo_aarch64 = "" - epol_repo_x86 = "" - if 'openEuler:Testing' in proj: - file_msg = """<project name="{0}"> - <title/> - <description/> - <person userid="Admin" role="maintainer"/> - <useforbuild> - <disable/> - </useforbuild> - <repository name="standard_aarch64"> - <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_aarch64"/> - <arch>aarch64</arch> - </repository> - <repository name="standard_x86_64"> - <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_x86_64"/> - <arch>x86_64</arch> - </repository> -</project> - """.format(proj) - else: - file_msg = """<project name="{0}"> - <title/> - <description/> - <person userid="Admin" role="maintainer"/> - <useforbuild> - <disable/> - </useforbuild> - <repository name="standard_aarch64"> - <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_aarch64"/>{3} - <arch>aarch64</arch> - </repository> - <repository name="standard_x86_64"> - <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_x86_64"/>{4} - <arch>x86_64</arch> - </repository> -</project> - """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) - f = open(file_path, "w") - f.write(file_msg) - f.close() - - def _write_selfbuild_meta_file(self, file_path, proj): - """ - write selfbuild project meta file - """ - file_msg = """<project name="{0}:selfbuild:BaseOS"> - <title/> - <description/> - <person userid="Admin" role="maintainer"/> - <repository name="{1}_standard_aarch64"> - <arch>aarch64</arch> - </repository> - <repository name="{1}_standard_x86_64"> - <arch>x86_64</arch> - </repository> - <repository name="{1}_epol_aarch64"> - <arch>aarch64</arch> - </repository> - <repository name="{1}_epol_x86_64"> - <arch>x86_64</arch> - </repository> -</project> -""".format(proj, proj.lower().replace(':', '_')) - f = open(file_path, "w") - f.write(file_msg) - f.close() - - def _add_pkg_service(self, tmp): - """ - add obs_meta packages _service file - """ - if 'Multi-Version' in tmp['branch_from']: - dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) - from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) - pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if tmp['branch_from'] == "master": - branch = "openEuler" - else: - branch = tmp['branch_from'] - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) - if os.system(cmd) == 0: - log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.error("copy %s service file failed!" % tmp['pkgname']) - - def _move_master_pkg_service(self, msg): - """ - move branch master obs_meta packages _service file - """ - # ****************************************** - # from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - # pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - # mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) - # if not os.path.exists(mv_to_path): - # os.makedirs(mv_to_path) - # if not os.path.exists(pkg_path): - # cmd = "mv %s %s" % (from_pkg_path,mv_to_path) - # if os.system(cmd) == 0: - # log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - # else: - # log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - # ******************************************** - change_pkgs = [] - for tmp in msg: - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - change_pkgs.append(tmp['pkgname']) - log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - else: - log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - return change_pkgs - - - def _add_prj_meta_pkgs_service(self, msg, branch_infos): - for tmp in msg: - if 'Multi-Version' not in tmp['branch_to']: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - if 'master' not in branch_infos: - self._add_pkg_service(tmp) - - def _del_pkg(self, tmp): - """ - delete obs_meta packages - """ - if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) - pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - if os.path.exists(pkg_path): - shutil.rmtree(pkg_path) - log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - return tmp['pkgname'] - - def _verify_meta_file(self, prj_pkg): - """ - verify obs_meta with pckg-mgmt.yaml - """ - for proj, pkg in prj_pkg.items(): - if proj != "branch": - if 'Multi-Version' in prj_pkg['branch']: - dir_name = '{}/{}'.format(prj_pkg['branch'],proj) - meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) - meta_pkglist = os.listdir(meta_path) - if 'README.md' in meta_pkglist: - meta_pkglist.remove('README.md') - else: - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) - need_del_pkg = set(meta_pkglist) - set(pkg) - log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) - for del_pkg in need_del_pkg: - tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} - self._del_pkg(tmp) - return list(need_del_pkg) - - def _push_code(self,repo): - """ - push code to gitee repo - """ - if os.path.exists(repo): - os.chdir(repo) - cmd = "git status -s" - if os.popen(cmd).read(): - cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" - if os.system(cmd) == 0: - cmd = "git push -f" - for i in range(5): - if os.system(cmd) == 0: - log.info("push code to gitee repo succeed!") - return 0 - else: - log.error("push code failed, try again...") - raise SystemExit("push code to gitee repo Failed!") - else: - log.info("No change, nothing to commit!") - return "nothing to push" - else: - log.error("%s not exist!" % repo) - return -1 - - def _dispose_msg_packages(self, msg, branch_infos): - ''' - add obs_meta packages _service file from new version yaml file - ''' - change_pkgs = [] - dir_type = branch_infos[1] - if dir_type == 'delete': - for tmp in msg: - pkg_name = self._del_pkg(tmp) - if pkg_name: - change_pkgs.append(pkg_name) - else: - for tmp in msg: - if 'Multi-Version' in tmp['branch_from']: - dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) - from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) - pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if tmp['branch_from'] == "master": - branch = "openEuler" - else: - branch = tmp['branch_from'] - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) - if os.system(cmd) == 0: - change_pkgs.append(tmp['pkgname']) - log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.error("copy %s service file failed!" % tmp['pkgname']) - return change_pkgs - - def _write_release_yaml(self,change_pkgs,branch): - ''' - write change info to release_change.yaml - ''' - if change_pkgs: - change_str = " ".join(change_pkgs) - os.chdir(self.release_management_path) - commit_cmd = 'git rev-parse HEAD' - commitid = os.popen(commit_cmd).read().split('\n')[0] - content_cmd = "git log --oneline -1" - content = os.popen(content_cmd).read().split('\n')[0] - reg=re.compile(r"(?<=!)\d+") - match=reg.search(content) - if match: - pr_id = match.group(0) - pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) - datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') - release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") - with open(release_change_yaml) as file: - result = yaml.load(file, Loader=yaml.FullLoader) - change_dic = { - 'pr':pull_request, - 'description':content, - 'changed_packages':change_str, - 'date':datestr - } - result['release-change'].append(change_dic) - with open(release_change_yaml, "w", encoding='utf-8') as f: - yaml.dump(result, f, default_flow_style=False, sort_keys=False) - log.info("write release change yaml file success") - else: - pull_request = commitid - log.info("ignore write release change yaml file success") - - - def sync_yaml_meta(self): - """ - integration of functions - """ - change_file = self._get_change_file() - master_change_file = [] - yaml_dict = {} - for line in change_file: - log.info("line:%s" % line) - name = list(line.split())[1] - file_path = os.path.join(self.release_management_path, name) - yaml_dict = self._get_yaml_file_msg(file_path) - branch_infos = name.split('/') - if 'master' in branch_infos: - branch = name.split('/')[1] - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") - self._add_prj_meta_pkgs_service(msg,branch_infos) - move_change_pkgs = self._move_master_pkg_service(msg) - del_change_pkgs = self._verify_meta_file(prj_pkg) - change_pkgs = move_change_pkgs + del_change_pkgs - self._write_release_yaml(change_pkgs,branch_infos[0]) - else: - if not yaml_dict: - log.info("%s file content is empty!" % name) - elif isinstance(yaml_dict['packages'], list): - msg = self._parse_yaml_msg_new(yaml_dict) - change_pkgs=self._dispose_msg_packages(msg,branch_infos) - self._write_release_yaml(change_pkgs,branch_infos[0]) - else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg, branch_infos) - else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg, branch_infos) - for tmp in del_msg: - self._del_pkg(tmp) - del_change_pkgs = self._verify_meta_file(prj_pkg) - ret = self._push_code(self.obs_meta_path) - self._push_code(self.release_management_path) - return ret - - -if __name__ == "__main__": - # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} - kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} - mgmt = SyncPckgMgmt(**kw) - mgmt.sync_yaml_meta() +#!/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: wangchong +# Create: 2021-06-08 +# ****************************************************************************** +""" +Synchronize the obs_meta file according to the pckg-mgmt.yaml file +""" + +import os +import re +import sys +import yaml +import shutil +import datetime +now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) +sys.path.append(os.path.join(now_path, "..")) +from common.log_obs import log + + +class SyncPckgMgmt(object): + """ + keep pckg-mgmt.yaml and obs_meta in sync + """ + def __init__(self, **kwargs): + """ + giteeuser: gitee user name + giteeuserpwd: gitee password + """ + self.kwargs = kwargs + self.giteeuser = self.kwargs['gitee_user'] + self.giteeuserpwd = self.kwargs['gitee_pwd'] + self.obs_meta_path = self.kwargs['obs_meta_path'] + self.release_management_path = self.kwargs['release_management_path'] + + def _get_change_file(self): + """ + get release-managemnet change file + """ + if os.path.exists(self.release_management_path): + os.chdir(self.release_management_path) + cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" + result = os.popen(cmd).read().split('\n') + change_file = [x for x in result if x != ''] + return change_file + else: + log.error("%s not exist!" % self.release_management_path) + sys.exit(1) + + def _get_yaml_file_msg(self, file_path): + """ + get pckg-mgmt.yaml file msg + """ + file_msg = None + if os.path.exists(file_path): + with open(file_path, "r", encoding='utf-8') as f: + file_msg = yaml.load(f, Loader=yaml.FullLoader) + return file_msg + + def _check_pckg_yaml(self, file_msg, branch): + """ + check pckg-mgmt.yaml file + """ + proj = branch.replace("-", ":") + flag = False + for title in file_msg['packages']: + for msg in file_msg['packages'][title]: + if msg['branch_to'] != branch: + flag = True + log.error("%s branch_to is error, please check yaml." % msg['name']) + if not msg['obs_to'].startswith(proj): + flag = True + log.error("%s obs_to is error, please check yaml." % msg['name']) + return flag + + def _parse_yaml_msg(self, file_msg, yaml_type): + """ + parse yaml file msg + """ + tmp = {} + msg = [] + del_msg = [] + prj_pkg = {} + if yaml_type == "old": + for pckg in file_msg['packages']['natural']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) + else: + for label in file_msg['packages']['everything']: + for pckg in file_msg['packages']['everything'][label]: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + for pckg in file_msg['packages']['epol']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + if yaml_type != 'master': + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + return msg, del_msg, prj_pkg + + def _parse_yaml_msg_new(self, file_msg): + """ + parse new version yaml file msg + """ + msg = [] + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + msg.append(tmp) + return msg + + def _write_prj_meta_file(self, file_path, proj): + """ + write project meta file + """ + if "Epol" in proj: + main_proj = proj.replace(':Epol', '') + con_proj = main_proj.replace(':', '_').lower() + epol_repo_aarch64 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_aarch64\"/>" % ( + main_proj, con_proj) + epol_repo_x86 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_x86_64\"/>" % ( + main_proj, con_proj) + else: + main_proj = proj + con_proj = main_proj.replace(':', '_').lower() + epol_repo_aarch64 = "" + epol_repo_x86 = "" + if 'openEuler:Testing' in proj: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <useforbuild> + <disable/> + </useforbuild> + <repository name="standard_aarch64"> + <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64"> + <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <useforbuild> + <disable/> + </useforbuild> + <repository name="standard_aarch64"> + <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_aarch64"/>{3} + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64"> + <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_x86_64"/>{4} + <arch>x86_64</arch> + </repository> +</project> + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + f = open(file_path, "w") + f.write(file_msg) + f.close() + + def _write_selfbuild_meta_file(self, file_path, proj): + """ + write selfbuild project meta file + """ + file_msg = """<project name="{0}:selfbuild:BaseOS"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="{1}_standard_aarch64"> + <arch>aarch64</arch> + </repository> + <repository name="{1}_standard_x86_64"> + <arch>x86_64</arch> + </repository> + <repository name="{1}_epol_aarch64"> + <arch>aarch64</arch> + </repository> + <repository name="{1}_epol_x86_64"> + <arch>x86_64</arch> + </repository> +</project> +""".format(proj, proj.lower().replace(':', '_')) + f = open(file_path, "w") + f.write(file_msg) + f.close() + + def _add_pkg_service(self, tmp): + """ + add obs_meta packages _service file + """ + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + + def _move_master_pkg_service(self, msg): + """ + move branch master obs_meta packages _service file + """ + # ****************************************** + # from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + # pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + # mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + # if not os.path.exists(mv_to_path): + # os.makedirs(mv_to_path) + # if not os.path.exists(pkg_path): + # cmd = "mv %s %s" % (from_pkg_path,mv_to_path) + # if os.system(cmd) == 0: + # log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + # else: + # log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + # ******************************************** + change_pkgs = [] + for tmp in msg: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return change_pkgs + + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): + for tmp in msg: + if 'Multi-Version' not in tmp['branch_to']: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'master' not in branch_infos: + self._add_pkg_service(tmp) + + def _del_pkg(self, tmp): + """ + delete obs_meta packages + """ + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] + + def _verify_meta_file(self, prj_pkg): + """ + verify obs_meta with pckg-mgmt.yaml + """ + for proj, pkg in prj_pkg.items(): + if proj != "branch": + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + need_del_pkg = set(meta_pkglist) - set(pkg) + log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) + for del_pkg in need_del_pkg: + tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} + self._del_pkg(tmp) + return list(need_del_pkg) + + def _push_code(self,repo): + """ + push code to gitee repo + """ + if os.path.exists(repo): + os.chdir(repo) + cmd = "git status -s" + if os.popen(cmd).read(): + cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" + if os.system(cmd) == 0: + cmd = "git push -f" + for i in range(5): + if os.system(cmd) == 0: + log.info("push code to gitee repo succeed!") + return 0 + else: + log.error("push code failed, try again...") + raise SystemExit("push code to gitee repo Failed!") + else: + log.info("No change, nothing to commit!") + return "nothing to push" + else: + log.error("%s not exist!" % repo) + return -1 + + def _dispose_msg_packages(self, msg, branch_infos): + ''' + add obs_meta packages _service file from new version yaml file + ''' + change_pkgs = [] + dir_type = branch_infos[1] + if dir_type == 'delete': + for tmp in msg: + pkg_name = self._del_pkg(tmp) + if pkg_name: + change_pkgs.append(pkg_name) + else: + for tmp in msg: + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + return change_pkgs + + def _write_release_yaml(self,change_pkgs,branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + else: + pull_request = commitid + log.info("ignore write release change yaml file success") + + + def sync_yaml_meta(self): + """ + integration of functions + """ + change_file = self._get_change_file() + master_change_file = [] + yaml_dict = {} + for line in change_file: + log.info("line:%s" % line) + name = list(line.split())[1] + file_path = os.path.join(self.release_management_path, name) + yaml_dict = self._get_yaml_file_msg(file_path) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = name.split('/')[1] + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + self._add_prj_meta_pkgs_service(msg,branch_infos) + move_change_pkgs = self._move_master_pkg_service(msg) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = move_change_pkgs + del_change_pkgs + self._write_release_yaml(change_pkgs,branch_infos[0]) + else: + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + msg = self._parse_yaml_msg_new(yaml_dict) + change_pkgs=self._dispose_msg_packages(msg,branch_infos) + self._write_release_yaml(change_pkgs,branch_infos[0]) + else: + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + self._add_prj_meta_pkgs_service(msg, branch_infos) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + self._add_prj_meta_pkgs_service(msg, branch_infos) + for tmp in del_msg: + self._del_pkg(tmp) + del_change_pkgs = self._verify_meta_file(prj_pkg) + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) + return ret + + +if __name__ == "__main__": + # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} + mgmt = SyncPckgMgmt(**kw) + mgmt.sync_yaml_meta() -- Gitee From 156f54de4f628f334b42f98505845a32d4ec9571 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 21 Jun 2022 10:46:00 +0800 Subject: [PATCH 08/10] wdw --- core/sync_pckg_mgmt.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 53b7314..3060070 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -164,7 +164,26 @@ class SyncPckgMgmt(object): con_proj = main_proj.replace(':', '_').lower() epol_repo_aarch64 = "" epol_repo_x86 = "" - file_msg = """<project name="{0}"> + if 'openEuler:Testing' in proj: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <useforbuild> + <disable/> + </useforbuild> + <repository name="standard_aarch64"> + <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64"> + <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> <title/> <description/> <person userid="Admin" role="maintainer"/> @@ -180,7 +199,7 @@ class SyncPckgMgmt(object): <arch>x86_64</arch> </repository> </project> -""".format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) f = open(file_path, "w") f.write(file_msg) f.close() -- Gitee From f1d844321b41eff1f445d9390f6bcd5eedfceb43 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 21 Jun 2022 17:02:56 +0800 Subject: [PATCH 09/10] ss --- core/package_manager.py | 2 +- core/sync_pckg_mgmt.py | 18 +----------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/core/package_manager.py b/core/package_manager.py index 4a2fe92..e9246b1 100755 --- a/core/package_manager.py +++ b/core/package_manager.py @@ -316,7 +316,7 @@ class OBSPkgManager(object): for line in data: file_content.append(line.strip('\n')) data.close() - with ThreadPoolExecutor(10) as executor: + with ThreadPoolExecutor(50) as executor: for content in file_content: executor.submit(self._obs_pkg_action, content) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 3060070..ecdf55c 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -169,9 +169,6 @@ class SyncPckgMgmt(object): <title/> <description/> <person userid="Admin" role="maintainer"/> - <useforbuild> - <disable/> - </useforbuild> <repository name="standard_aarch64"> <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_aarch64"/> <arch>aarch64</arch> @@ -264,21 +261,8 @@ class SyncPckgMgmt(object): def _move_master_pkg_service(self, msg): """ - move branch master obs_meta packages _service file + copy branch master obs_meta packages _service file """ - # ****************************************** - # from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - # pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - # mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) - # if not os.path.exists(mv_to_path): - # os.makedirs(mv_to_path) - # if not os.path.exists(pkg_path): - # cmd = "mv %s %s" % (from_pkg_path,mv_to_path) - # if os.system(cmd) == 0: - # log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - # else: - # log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - # ******************************************** change_pkgs = [] for tmp in msg: from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) -- Gitee From fe69152142e01367c3a9088bb9c3cc7435cd1786 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 25 Jun 2022 13:21:05 +0800 Subject: [PATCH 10/10] ddd --- config/config.ini | 8 ++++++-- core/sync_pckg_mgmt.py | 2 +- tools/create_master_pckg_mgmt_yaml.py | 6 ++++-- tools/modify_src_openeuler_yaml.py | 1 - 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/config/config.ini b/config/config.ini index f00c9ea..67a17f9 100644 --- a/config/config.ini +++ b/config/config.ini @@ -30,7 +30,7 @@ name = ci_check build [package_info_file] name = package_info.csv [branch_proj] -master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras openEuler:Testing:BaseTools openEuler:Testing:C openEuler:Testing:Common_Languages_Dependent_Tools openEuler:Testing:Epol openEuler:Testing:Erlang openEuler:Testing:Factory openEuler:Testing:Java openEuler:Testing:KernelSpace openEuler:Testing:Mainline openEuler:Testing:Meson openEuler:Testing:MultiLanguage openEuler:Testing:Nodejs openEuler:Testing:Perl openEuler:Testing:Python openEuler:Testing:Ruby +master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras openEuler:Testing:BaseTools openEuler:Testing:C openEuler:Testing:Common_Languages_Dependent_Tools openEuler:Testing:Epol openEuler:Testing:Erlang openEuler:Testing:Factory openEuler:Testing:Golang openEuler:Testing:Java openEuler:Testing:KernelSpace openEuler:Testing:Lua openEuler:Testing:Mainline openEuler:Testing:Meson openEuler:Testing:MultiLanguage openEuler:Testing:Nodejs openEuler:Testing:Ocaml openEuler:Testing:Perl openEuler:Testing:Python openEuler:Testing:Qt openEuler:Testing:Ruby openEuler-20.03-LTS = openEuler:20.03:LTS openEuler:20.03:LTS:Epol openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1 openEuler:20.03:LTS:SP1:Epol openEuler:20.03:LTS:SP1:Extras openEuler-20.03-LTS-Next = openEuler:20.03:LTS:Next openEuler:20.03:LTS:Next:Epol openEuler:20.03:LTS:Next:Extras @@ -71,14 +71,18 @@ openEuler-Testing-Common_Languages_Dependent_Tools = openEuler:Testing:selfbuild openEuler-Testing-Epol = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Erlang = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Factory = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Golang = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Java = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-KernelSpace = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Lua = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Mainline = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Meson = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-MultiLanguage = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Nodejs = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Ocaml = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Perl = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Python = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Qt = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-Testing-Ruby = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-20.03-LTS = openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_aarch64 openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_x86_64 openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_x86_64 @@ -120,4 +124,4 @@ path = /srv/obs/build [obs_ignore_package] name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv [obs_include_project] -name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:Testing:BaseTools openEuler:Testing:C openEuler:Testing:Common_Languages_Dependent_Tools openEuler:Testing:Epol openEuler:Testing:Erlang openEuler:Testing:Factory openEuler:Testing:Java openEuler:Testing:KernelSpace openEuler:Testing:Mainline openEuler:Testing:Meson openEuler:Testing:MultiLanguage openEuler:Testing:Nodejs openEuler:Testing:Perl openEuler:Testing:Python openEuler:Testing:Ruby \ No newline at end of file +name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:Testing:BaseTools openEuler:Testing:C openEuler:Testing:Common_Languages_Dependent_Tools openEuler:Testing:Epol openEuler:Testing:Erlang openEuler:Testing:Factory openEuler:Testing:Golang openEuler:Testing:Java openEuler:Testing:KernelSpace openEuler:Testing:Lua openEuler:Testing:Mainline openEuler:Testing:Meson openEuler:Testing:MultiLanguage openEuler:Testing:Nodejs openEuler:Testing:Ocaml openEuler:Testing:Perl openEuler:Testing:Python openEuler:Testing:Qt openEuler:Testing:Ruby \ No newline at end of file diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index ecdf55c..d243b43 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -483,4 +483,4 @@ if __name__ == "__main__": # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} mgmt = SyncPckgMgmt(**kw) - mgmt.sync_yaml_meta() + mgmt.sync_yaml_meta() \ No newline at end of file diff --git a/tools/create_master_pckg_mgmt_yaml.py b/tools/create_master_pckg_mgmt_yaml.py index 477722e..6382bce 100644 --- a/tools/create_master_pckg_mgmt_yaml.py +++ b/tools/create_master_pckg_mgmt_yaml.py @@ -43,10 +43,12 @@ def parse_pkgs(all_dict): # pkgs_dict['packages'] = pkgs if branch in ['openEuler-Factory','openEuler-Epol']: obs_from = branch.replace("-",":") - obs_to = "{}:Testing:{}".format(branch.split("-")[0],branch.split("-")[1]) + obs_to = branch.replace("-",":") + # obs_to = "{}:Testing:{}".format(branch.split("-")[0],branch.split("-")[1]) else: obs_from = 'openEuler:Mainline' - obs_to = "{}:Testing:{}".format(branch.split("-")[0],branch.split("-")[1]) + obs_to = branch.replace("-",":") + # obs_to = "{}:Testing:{}".format(branch.split("-")[0],branch.split("-")[1]) for pkg in pkgs: pkgs_dict["packages"].append({ "name": pkg, diff --git a/tools/modify_src_openeuler_yaml.py b/tools/modify_src_openeuler_yaml.py index 63420f2..112da29 100644 --- a/tools/modify_src_openeuler_yaml.py +++ b/tools/modify_src_openeuler_yaml.py @@ -23,7 +23,6 @@ import yaml import shutil import argparse - par = argparse.ArgumentParser() par.add_argument("-pm", "--pckg_mgmt", default=None, help="pckg_mgmt.yaml file path", required=True) -- Gitee