diff --git a/config/config.ini b/config/config.ini index 5ae97f32b208ac6ccf8f7757f6e3e698f982dfd5..67a17f9281d815a064eabaa7d96ce0c197a728a2 100644 --- a/config/config.ini +++ b/config/config.ini @@ -30,7 +30,7 @@ name = ci_check build [package_info_file] name = package_info.csv [branch_proj] -master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras +master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras openEuler:Testing:BaseTools openEuler:Testing:C openEuler:Testing:Common_Languages_Dependent_Tools openEuler:Testing:Epol openEuler:Testing:Erlang openEuler:Testing:Factory openEuler:Testing:Golang openEuler:Testing:Java openEuler:Testing:KernelSpace openEuler:Testing:Lua openEuler:Testing:Mainline openEuler:Testing:Meson openEuler:Testing:MultiLanguage openEuler:Testing:Nodejs openEuler:Testing:Ocaml openEuler:Testing:Perl openEuler:Testing:Python openEuler:Testing:Qt openEuler:Testing:Ruby openEuler-20.03-LTS = openEuler:20.03:LTS openEuler:20.03:LTS:Epol openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1 openEuler:20.03:LTS:SP1:Epol openEuler:20.03:LTS:SP1:Extras openEuler-20.03-LTS-Next = openEuler:20.03:LTS:Next openEuler:20.03:LTS:Next:Epol openEuler:20.03:LTS:Next:Extras @@ -65,6 +65,25 @@ obs_pkg_rpms = https://gitee.com/unsunghero/obs_pkg_rpms openEuler-Mainline = openEuler:selfbuild:BaseOS/mainline_standard_aarch64 openEuler:selfbuild:BaseOS/mainline_standard_x86_64 openEuler-Epol = openEuler:selfbuild:BaseOS/epol_aarch64 openEuler:selfbuild:BaseOS/epol_x86_64 openEuler-Factory = openEuler:selfbuild:BaseOS/factory_aarch64 openEuler:selfbuild:BaseOS/factory_x86_64 +openEuler-Testing-BaseTools = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-C = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Common_Languages_Dependent_Tools = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Epol = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Erlang = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Factory = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Golang = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Java = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-KernelSpace = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Lua = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Mainline = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Meson = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-MultiLanguage = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Nodejs = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Ocaml = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Perl = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Python = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Qt = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 +openEuler-Testing-Ruby = openEuler:Testing:selfbuild:BaseOS/baseos_aarch64 openEuler:Testing:selfbuild:BaseOS/baseos_x86_64 openEuler-20.03-LTS = openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_aarch64 openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_x86_64 openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_x86_64 openEuler-20.03-LTS-SP1-Epol = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_x86_64 @@ -105,4 +124,4 @@ path = /srv/obs/build [obs_ignore_package] name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv [obs_include_project] -name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely \ No newline at end of file +name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:Testing:BaseTools openEuler:Testing:C openEuler:Testing:Common_Languages_Dependent_Tools openEuler:Testing:Epol openEuler:Testing:Erlang openEuler:Testing:Factory openEuler:Testing:Golang openEuler:Testing:Java openEuler:Testing:KernelSpace openEuler:Testing:Lua openEuler:Testing:Mainline openEuler:Testing:Meson openEuler:Testing:MultiLanguage openEuler:Testing:Nodejs openEuler:Testing:Ocaml openEuler:Testing:Perl openEuler:Testing:Python openEuler:Testing:Qt openEuler:Testing:Ruby \ No newline at end of file diff --git a/core/package_manager.py b/core/package_manager.py index 4a2fe92657477cc6ddf3c423e69d53e7e7351e1e..e9246b1984d0c3a67d3f91f836353d00a037e5ff 100755 --- a/core/package_manager.py +++ b/core/package_manager.py @@ -316,7 +316,7 @@ class OBSPkgManager(object): for line in data: file_content.append(line.strip('\n')) data.close() - with ThreadPoolExecutor(10) as executor: + with ThreadPoolExecutor(50) as executor: for content in file_content: executor.submit(self._obs_pkg_action, content) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index e0e18d9ee9082e7512ae29268eb21a9ab833cda3..d243b437af20380d3ba2e60bf640ac56e3d90fb9 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -18,9 +18,11 @@ Synchronize the obs_meta file according to the pckg-mgmt.yaml file """ import os +import re import sys import yaml import shutil +import datetime now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(os.path.join(now_path, "..")) from common.log_obs import log @@ -97,6 +99,14 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) else: for label in file_msg['packages']['everything']: for pckg in file_msg['packages']['everything'][label]: @@ -113,17 +123,30 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) - for pckg in file_msg['packages']['recycle']: + if yaml_type != 'master': + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + return msg, del_msg, prj_pkg + + def _parse_yaml_msg_new(self, file_msg): + """ + parse new version yaml file msg + """ + msg = [] + for pckg in file_msg['packages']: tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) - return msg, del_msg, prj_pkg + return msg def _write_prj_meta_file(self, file_path, proj): """ @@ -141,7 +164,23 @@ class SyncPckgMgmt(object): con_proj = main_proj.replace(':', '_').lower() epol_repo_aarch64 = "" epol_repo_x86 = "" - file_msg = """ + if 'openEuler:Testing' in proj: + file_msg = """ + + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="standard_aarch64"> + <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64"> + <path project="openEuler:Testing:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> <title/> <description/> <person userid="Admin" role="maintainer"/> @@ -157,7 +196,7 @@ class SyncPckgMgmt(object): <arch>x86_64</arch> </repository> </project> -""".format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) f = open(file_path, "w") f.write(file_msg) f.close() @@ -192,8 +231,16 @@ class SyncPckgMgmt(object): """ add obs_meta packages _service file """ - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) pkg_service_path = os.path.join(pkg_path, "_service") if tmp['branch_from'] == "master": branch = "openEuler" @@ -211,31 +258,59 @@ class SyncPckgMgmt(object): log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.error("copy %s service file failed!" % tmp['pkgname']) - - def _add_prj_meta_pkgs_service(self, msg): + + def _move_master_pkg_service(self, msg): + """ + copy branch master obs_meta packages _service file + """ + change_pkgs = [] for tmp in msg: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to']: - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - self._add_pkg_service(tmp) + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return change_pkgs + + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): + for tmp in msg: + if 'Multi-Version' not in tmp['branch_to']: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'master' not in branch_infos: + self._add_pkg_service(tmp) def _del_pkg(self, tmp): """ delete obs_meta packages """ - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) if os.path.exists(pkg_path): shutil.rmtree(pkg_path) log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] def _verify_meta_file(self, prj_pkg): """ @@ -243,19 +318,27 @@ class SyncPckgMgmt(object): """ for proj, pkg in prj_pkg.items(): if proj != "branch": - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) need_del_pkg = set(meta_pkglist) - set(pkg) log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) for del_pkg in need_del_pkg: tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} self._del_pkg(tmp) + return list(need_del_pkg) - def _push_code(self): + def _push_code(self,repo): """ push code to gitee repo """ - if os.path.exists(self.obs_meta_path): - os.chdir(self.obs_meta_path) + if os.path.exists(repo): + os.chdir(repo) cmd = "git status -s" if os.popen(cmd).read(): cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" @@ -272,38 +355,132 @@ class SyncPckgMgmt(object): log.info("No change, nothing to commit!") return "nothing to push" else: - log.error("%s not exist!" % self.obs_meta_path) + log.error("%s not exist!" % repo) return -1 + def _dispose_msg_packages(self, msg, branch_infos): + ''' + add obs_meta packages _service file from new version yaml file + ''' + change_pkgs = [] + dir_type = branch_infos[1] + if dir_type == 'delete': + for tmp in msg: + pkg_name = self._del_pkg(tmp) + if pkg_name: + change_pkgs.append(pkg_name) + else: + for tmp in msg: + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + return change_pkgs + + def _write_release_yaml(self,change_pkgs,branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + else: + pull_request = commitid + log.info("ignore write release change yaml file success") + + def sync_yaml_meta(self): """ integration of functions """ change_file = self._get_change_file() + master_change_file = [] yaml_dict = {} for line in change_file: log.info("line:%s" % line) name = list(line.split())[1] - branch = name.split('/')[0] file_path = os.path.join(self.release_management_path, name) yaml_dict = self._get_yaml_file_msg(file_path) - if not yaml_dict: - log.info("%s file content is empty!" % name) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = name.split('/')[1] + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + self._add_prj_meta_pkgs_service(msg,branch_infos) + move_change_pkgs = self._move_master_pkg_service(msg) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = move_change_pkgs + del_change_pkgs + self._write_release_yaml(change_pkgs,branch_infos[0]) else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg) + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + msg = self._parse_yaml_msg_new(yaml_dict) + change_pkgs=self._dispose_msg_packages(msg,branch_infos) + self._write_release_yaml(change_pkgs,branch_infos[0]) else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg) - for tmp in del_msg: - self._del_pkg(tmp) - self._verify_meta_file(prj_pkg) - ret = self._push_code() + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + self._add_prj_meta_pkgs_service(msg, branch_infos) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + self._add_prj_meta_pkgs_service(msg, branch_infos) + for tmp in del_msg: + self._del_pkg(tmp) + del_change_pkgs = self._verify_meta_file(prj_pkg) + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) return ret if __name__ == "__main__": - kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} mgmt = SyncPckgMgmt(**kw) - mgmt.sync_yaml_meta() + mgmt.sync_yaml_meta() \ No newline at end of file diff --git a/tools/create_master_pckg_mgmt_yaml.py b/tools/create_master_pckg_mgmt_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..6382bce20a213bd0962e0ffc94c2b331601beb8b --- /dev/null +++ b/tools/create_master_pckg_mgmt_yaml.py @@ -0,0 +1,79 @@ +#/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: dongjie +# Create: 2022-06-16 +# ****************************************************************************** +""" +create pckg-mgmt.yaml +""" + +import argparse +import os +import sys +import yaml +import datetime +import pandas as pd + +# all_dict = {"openEuler-Factory":['A','B'],"openEuler-MultiLanguage":['C','D']} +# pkgs_dict = {"packages": {"everything": {"baseos": [], "other": []},"epol": [], "recycle": [], "delete": []}} +datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + +def write_yaml(dict_msg, file_path): + with open(file_path, "w", encoding='utf-8') as f: + yaml.dump(dict_msg, f, default_flow_style=False, sort_keys=False) + +def parse_pkgs(all_dict): + for branch,pkgs in all_dict.items(): + print (branch) + pkgs_dict = {"packages":[]} + yaml_path = os.path.join(os.getcwd(),"{}/pckg-mgmt.yaml".format(branch)) + dir_path = os.path.join(os.getcwd(),branch) + if not os.path.isdir(dir_path): + os.makedirs(dir_path) + # pkgs_dict['packages'] = pkgs + if branch in ['openEuler-Factory','openEuler-Epol']: + obs_from = branch.replace("-",":") + obs_to = branch.replace("-",":") + # obs_to = "{}:Testing:{}".format(branch.split("-")[0],branch.split("-")[1]) + else: + obs_from = 'openEuler:Mainline' + obs_to = branch.replace("-",":") + # obs_to = "{}:Testing:{}".format(branch.split("-")[0],branch.split("-")[1]) + for pkg in pkgs: + pkgs_dict["packages"].append({ + "name": pkg, + # "branch_from": from_branch, + # "branch_to": to_branch, + "obs_from": obs_from, + "obs_to": obs_to, + "date": datestr + }) + write_yaml(pkgs_dict,yaml_path) + +def parse_excel(excel_path): + all_dict = {} + df = pd.read_excel(excel_path,sheet_name=1,header =0) + for index,row in df.iterrows(): + project = row[1].replace(":","-") + pkg = row[0] + if all_dict.get(project,''): + all_dict[project].append(pkg) + else: + all_dict[project] = [pkg] + return all_dict + + +if __name__ == '__main__': + excel_path = r'D:\分层构建\layer.xlsx' + all_dict = parse_excel(excel_path) + parse_pkgs(all_dict) \ No newline at end of file diff --git a/tools/modify_src_openeuler_yaml.py b/tools/modify_src_openeuler_yaml.py index 63420f285f49fa6bcdbdae9829d52e0e81d81397..112da291effbe4e62caf0cad87a807bf871effd9 100644 --- a/tools/modify_src_openeuler_yaml.py +++ b/tools/modify_src_openeuler_yaml.py @@ -23,7 +23,6 @@ import yaml import shutil import argparse - par = argparse.ArgumentParser() par.add_argument("-pm", "--pckg_mgmt", default=None, help="pckg_mgmt.yaml file path", required=True)