From b7e8710199c2d12e437b24dba1fd86d2a2343851 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 20 Jul 2022 09:53:51 +0800 Subject: [PATCH 01/19] =?UTF-8?q?=E5=88=86=E5=B1=82=E6=9E=84=E5=BB=BA&?= =?UTF-8?q?=E5=88=86=E6=94=AFyaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/config.ini | 22 +- core/check_release_management.py | 606 +++++++++++++++++++- core/sync_pckg_mgmt.py | 931 ++++++++++++++++++++----------- 3 files changed, 1194 insertions(+), 365 deletions(-) diff --git a/config/config.ini b/config/config.ini index 922ee3f..b67934d 100644 --- a/config/config.ini +++ b/config/config.ini @@ -31,7 +31,7 @@ name = ci_check build [package_info_file] name = package_info.csv [branch_proj] -master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras +master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Erlang openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby openEuler-20.03-LTS = openEuler:20.03:LTS openEuler:20.03:LTS:Epol openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1 openEuler:20.03:LTS:SP1:Epol openEuler:20.03:LTS:SP1:Extras openEuler-20.03-LTS-Next = openEuler:20.03:LTS:Next openEuler:20.03:LTS:Next:Epol openEuler:20.03:LTS:Next:Extras @@ -64,9 +64,25 @@ obs_meta = https://gitee.com/src-openeuler/obs_meta obs_pkg_rpms = https://gitee.com/openeuler_latest_rpms/obs_pkg_rpms [obs_project_repos] # replace obs project's ":" by "-" -openEuler-Mainline = openEuler:selfbuild:BaseOS/mainline_standard_aarch64 openEuler:selfbuild:BaseOS/mainline_standard_x86_64 +openEuler-Mainline = oopenEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 openEuler-Epol = openEuler:selfbuild:BaseOS/epol_aarch64 openEuler:selfbuild:BaseOS/epol_x86_64 openEuler-Factory = openEuler:selfbuild:BaseOS/factory_aarch64 openEuler:selfbuild:BaseOS/factory_x86_64 +openEuler-BaseTools = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-C = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Common_Languages_Dependent_Tools = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Erlang = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Golang = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Java = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-KernelSpace = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Lua = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Meson = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-MultiLanguage = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Nodejs = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Ocaml = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Perl = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Python = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Qt = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Ruby = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 openEuler-20.03-LTS = openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_aarch64 openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_x86_64 openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_x86_64 openEuler-20.03-LTS-SP1-Epol = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_x86_64 @@ -109,4 +125,4 @@ path = /srv/obs/build [obs_ignore_package] name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv [obs_include_project] -name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely +name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Erlang openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby diff --git a/core/check_release_management.py b/core/check_release_management.py index 50f5877..589da52 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -118,19 +118,39 @@ class CheckReleaseManagement(object): get the change file for latest commit """ new_file_path = [] + new_versin_file_path = [] + master_new_file_path = [] + multi_version_file_path = [] for line in change_file: log.info("line:%s" % line) log_list = list(line.split()) temp_log_type = log_list[0] if len(log_list) == 3: if "pckg-mgmt.yaml" in log_list[2]: - new_file_path.append(log_list[2]) + if 'master' in log_list[2]: + master_new_file_path.append(log_list[2]) + elif 'multi_version' in log_list[2]: + master_new_file_path.append(log_list[2]) + else: + branch_infos = log_list[2].split('/') + if len(branch_infos) == 3: + new_versin_file_path.append(log_list[2]) + else: + new_file_path.append(log_list[2]) elif len(log_list) == 2: if temp_log_type != "D" and "pckg-mgmt.yaml" in log_list[1]: - new_file_path.append(log_list[1]) - if new_file_path: - log.info(new_file_path) - return new_file_path + if 'master' in log_list[1]: + master_new_file_path.append(log_list[1]) + elif 'multi_version' in log_list[1]: + master_new_file_path.append(log_list[1]) + else: + branch_infos = log_list[1].split('/') + if len(branch_infos) == 3: + new_versin_file_path.append(log_list[1]) + else: + new_file_path.append(log_list[1]) + if new_file_path or master_new_file_path or new_versin_file_path: + return new_file_path,master_new_file_path,new_versin_file_path else: log.info("There are no file need to check!!!") sys.exit() @@ -246,8 +266,7 @@ class CheckReleaseManagement(object): delete_tag = self._check_delete_tag(msg, yaml_all_msg[change]) if delete_tag: continue - msg_path = os.path.join(meta_path, msg['branch_from'], - msg['obs_from'], msg['name']) + msg_path = os.path.join(meta_path, msg['branch_from'],msg['obs_from'], msg['name']) if not os.path.exists(msg_path): yaml_key = os.path.join(msg['branch_from'], msg['obs_from'], msg['name']) @@ -404,35 +423,568 @@ class CheckReleaseManagement(object): log.error("Check the delete group in the {0}!!!".format(info_dict)) raise SystemExit("ERROR:Please check your PR") + def _get_move_and_add(self,old_msg,new_msg): + ''' + get master project add and delete pkgs by compare old and new + ''' + add_infos = {} + delete_infos = {} + for branch,old_pkgs in old_msg.items(): + if old_pkgs: + new_pkgs = new_msg[branch] + old_pkgs_names = [info['name'] for info in old_pkgs] + new_pkgs_names = [info['name'] for info in new_pkgs] + del_names = list(set(old_pkgs_names).difference(set(new_pkgs_names))) + add_names = list(set(new_pkgs_names).difference(set(old_pkgs_names))) + if del_names: + for old in old_pkgs: + if old['name'] in del_names: + if delete_infos.get(branch,[]): + delete_infos[branch].append(old) + else: + delete_infos[branch] = [old] + if add_names: + for new in new_pkgs: + if new['name'] in add_names: + if add_infos.get(branch,[]): + add_infos[branch].append(new) + else: + add_infos[branch] = [new] + else: + add_infos[branch] = new_msg[branch] + return add_infos,delete_infos + + def _check_master_add_rules(self, add_infos, move_infos): + ''' + check master add or internal move pkgs rules + ''' + if add_infos: + log.info("master add pkgs obs_from check") + error_infos = {} + error_flag = False + for branch,pkgs in add_infos.items(): + log.info('branch:{}'.format(branch)) + log.info('pkgs:{}'.format(pkgs)) + if branch == 'openEuler-Factory': + branch = branch.replace("-",":") + for pkg in pkgs: + if pkg['obs_to'] != branch: + error_flag = True + log.error("pkg name:{2} Wrong obs_to: in !!!".format(pkg['obs_to'],branch,pkg['name'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + else: + for pkg in pkgs: + if 'Multi-Version' in branch: + branch = branch.replace("_",":") + from_result = self._find_master_meta_path(pkg,ctype='multi-from') + move_result = False + else: + branch = branch.replace("-",":") + from_result = self._find_master_meta_path(pkg) + move_result = self._find_in_delete_pkgs(branch,pkg,move_infos) + if pkg['obs_to'] != branch: + error_flag = True + log.error("pkg name:{2} Wrong obs_to: in !!!".format(pkg['obs_to'],branch,pkg['name'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if from_result or move_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + def _find_in_delete_pkgs(self,branch,pkg,move_infos): + ''' + check move pkg in from branch yaml is deleted + ''' + flag = False + branchs = list(move_infos.keys()) + pkg_from = pkg['obs_from'].replace(":","-") + if pkg_from not in branchs: + log.error("pkg name:{} need delete in from branch:{} yaml!!!".format(pkg['name'],pkg['obs_from'])) + flag = True + else: + pkgnames = [info['name'] for info in move_infos[pkg_from]] + if pkg['name'] not in pkgnames: + log.error("pkg name:{} need delete in from branch:{} yaml!!!".format(pkg['name'],pkg['obs_from'])) + flag = True + return flag + + + def _check_master_del_rules(self, old_msg, new_msg): + ''' + check master delete pckg-mgmt.yaml pkgs is exist in obs_meta + ''' + error_flag = False + for branch,old_pkgs in old_msg.items(): + new_pkgs = new_msg[branch] + old_pkgs_names = [info['name'] for info in old_pkgs] + new_pkgs_names = [info['name'] for info in new_pkgs] + del_names = list(set(old_pkgs_names).difference(set(new_pkgs_names))) + add_names = list(set(new_pkgs_names).difference(set(old_pkgs_names))) + if add_names: + branch_dir = os.path.join(self.meta_path, 'master') + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in add_names and 'bringInRely' not in c_path and 'RISC-V' not in c_path: + add_names.remove(name) + if add_names: + error_flag = True + log.error("master branch pkg name:{} you want delete not exist in obs_meta!!!".format(add_names)) + return error_flag + + + def _find_master_meta_path(self, pkg, ctype='from'): + ''' + find obs_form or obs_to in obs_meta path + ''' + if ctype == 'from': + pkg_from_path = os.path.join(self.meta_path, 'master', pkg['obs_from'], pkg['name']) + if not os.path.exists(pkg_from_path): + yaml_key = os.path.join('master',pkg['obs_from'], pkg['name']) + log.error("The {0} not exist in obs_meta".format(yaml_key)) + return True + return False + elif ctype == 'multi-from': + if 'Multi-Version' in pkg['source_dir']: + dir_name = '{}/{}'.format(pkg['source_dir'], pkg['obs_from']) + pkg_from_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) + yaml_key = os.path.join('multi_version', dir_name, pkg['name']) + else: + pkg_from_path = os.path.join(self.meta_path, pkg['source_dir'], pkg['obs_from'], pkg['name']) + yaml_key = os.path.join(pkg['source_dir'], pkg['obs_from'], pkg['name']) + if not os.path.exists(pkg_from_path): + log.error("The {0} not exist in obs_meta".format(yaml_key)) + return True + return False + elif ctype == 'multi-to': + if 'Multi-Version' in pkg['destination_dir']: + dir_name = '{}/{}'.format(pkg['destination_dir'],pkg['obs_to']) + pkg_to_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) + yaml_key = os.path.join('multi_version', dir_name, pkg['name']) + else: + pkg_to_path = os.path.join(self.meta_path, pkg['destination_dir'], pkg['obs_to'], pkg['name']) + yaml_key = os.path.join(pkg['destination_dir'], pkg['obs_to'], pkg['name']) + if not os.path.exists(pkg_to_path): + log.error("The pkg {0} you want delete not exist in obs_meta".format(yaml_key)) + return True + return False + else: + pkg_to_path = os.path.join(self.meta_path, 'master', pkg['obs_to'], pkg['name']) + yaml_key = os.path.join('master', pkg['obs_to'], pkg['name']) + if not os.path.exists(pkg_to_path): + log.error("The pkg {0} you want move not exist in obs_meta".format(yaml_key)) + return True + return False + + def _check_master_move_rules(self, delete_infos): + ''' + check master branch internal move pkgs rule + ''' + if delete_infos: + log.info("master internal move pkgs check") + error_infos = {} + error_flag = False + for branch,pkgs in delete_infos.items(): + log.info('branch:{}'.format(branch)) + log.info('pkgs:{}'.format(pkgs)) + if branch == 'openEuler-Factory': + branch = branch.replace("-",":") + for pkg in pkgs: + to_result = self._find_master_meta_path(pkg, ctype='to') + else: + # branch = branch.replace("-",":") + for pkg in pkgs: + if 'Multi-Version' in branch: + branch = branch.replace("_",":") + to_result = self._find_master_meta_path(pkg, ctype='multi-to') + else: + branch = branch.replace("-",":") + to_result = self._find_master_meta_path(pkg, ctype='to') + if to_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + def _check_master_date_rules(self, infos): + ''' + check date is today + ''' + error_infos = {} + error_flag = False + date = datetime.date.today() + today = date.day + for branch,pkgs in infos.items(): + if branch != 'delete': + for pkg in pkgs: + log.info("master_branch:{},{} date check".format(branch,pkg['name'])) + yaml_date = int(pkg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error(pkg) + log.error("Wrong Date: !!!".format(pkg['date'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + + def _check_pkg_from_new(self, meta_path, change_info): + """ + check add pkg obs_from exist in obs_meta + """ + correct_from_check = {} + error_from_check = {} + error_flag = False + for branch,change in change_info.items(): + log.info("{} pkg obs_from check".format(branch)) + for msg in change: + # log.info("{} pkg obs_from check".format(msg['name'])) + msg_path = os.path.join(meta_path, msg['source_dir'],msg['obs_from'], msg['name']) + yaml_key = os.path.join(msg['source_dir'],msg['obs_from'], msg['name']) + if not os.path.exists(msg_path): + error_from_check.setdefault(branch,[]).append(msg) + else: + correct_from_check.setdefault(branch,[]).append(msg) + return correct_from_check,error_from_check + + def _check_pkg_delete_new(self, meta_path, change_info): + """ + check delete pkg exist in obs_meta + """ + error_flag = False + for branch,change in change_info.items(): + log.info("{} pkg obs_from check".format(branch)) + if change: + pkgs = [msg['name'] for msg in change] + log.info("The {0} exist in obs_meta dir {1} check".format(pkgs,branch)) + branch_dir = os.path.join(meta_path,branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path: + pkgs.remove(name) + if pkgs: + log.error("The {0} not exist in obs_meta dir {1}".format(pkgs,branch)) + error_flag = True + return error_flag + + + def _check_pkg_parent_from(self, change_info, correct_from, error_from, add_infos): + ''' + re-check error pkg_from and find in parent branch add + ''' + print (correct_from) + print (error_from) + error_flag = False + branch_msg_path = os.path.join(self.manage_path, "valid_release_branches.yaml") + with open(branch_msg_path, 'r', encoding='utf-8') as f: + branch_result = yaml.load(f, Loader=yaml.FullLoader) + all_master_add = {} + if add_infos: + for master_branch, master_pkgs in add_infos.items(): + master_pkgnames = [line['name'] for line in master_pkgs] + all_master_add[master_branch] = master_pkgnames + print (all_master_add) + if error_from: + for branch, pkgs in error_from.items(): + error_names = [pkg['name'] for pkg in pkgs] + print (error_names) + for pkg in pkgs: + master_branch_name = pkg['obs_from'].replace(":",'-') + if branch in branch_result['branch']['master'] and all_master_add.get(master_branch_name,''): + temps = all_master_add[master_branch_name] + if pkg['name'] in temps: + error_names.remove(pkg['name']) + else: + parent_branch = '' + for valied_parent,child_branchs in branch_result['branch'].items(): + if branch in child_branchs: + parent_branch = valied_parent + break + if parent_branch: + if correct_from.get(parent_branch,''): + correct_branch_names = [line['name'] for line in correct_from[parent_branch]] + if pkg['name'] in correct_branch_names: + error_names.remove(pkg['name']) + break + if error_names: + error_flag =True + log.error("branch:{},The {} not exist in obs_meta dir and not in parent branch obs_meta dir too".format(branch,error_names)) + return error_flag + + + def _check_key_in_yaml_new(self, change_info): + """ + check the key and brach from in your yaml compliance with rules + """ + error_flag = False + keylist = ['source_dir', 'obs_from', 'name', 'destination_dir', 'obs_to', 'date'] + for branch,info in change_info.items(): + if info: + log.info("branch:{} yaml key check".format(branch)) + for msg in info: + if len(msg.keys()) == 7 or len(msg.keys()) == 6: + for key in msg.keys(): + if key not in keylist: + error_flag = True + log.error(msg) + log.error("ERROR:<<<<<<{0}:>>>>>> should not in there".format(key)) + else: + error_flag = True + log.error("Please check {0}".format(msg)) + if error_flag: + raise SystemExit("ERROR: Please ensure the following key values in your yaml") + + def _check_valid_release_branch(self, change_info): + """ + check the source_dir and destination_dir in your yaml compliance with rules + """ + error_flag = False + branch_msg_path = os.path.join(self.manage_path, "valid_release_branches.yaml") + with open(branch_msg_path, 'r', encoding='utf-8') as f: + branch_result = yaml.load(f, Loader=yaml.FullLoader) + for branch,msg in change_info.items(): + log.info("{} source_dir and destination_dir valid check".format(branch)) + for pkg in msg: + if pkg['destination_dir'] == branch and \ + pkg['source_dir'] in branch_result['branch'].keys() and \ + pkg['destination_dir'] in branch_result['branch'][pkg['source_dir']]: + continue + else: + error_flag = True + log.error("pkg:{1} souce_dir or destination_dir valid check error".format(pkg['name'])) + if error_flag: + raise SystemExit("ERROR: Please ensure the source_dir and destination_dir adapt rules") + + def _check_pkg_date(self, change_info): + ''' + check new version pkgs date + ''' + error_flag = False + date = datetime.date.today() + today = date.day + for branch,msg in change_info.items(): + log.info("{0} date check".format(branch)) + for pkg in msg: + yaml_date = int(pkg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error("Wrong Date: !!!".format(pkg['date'])) + return error_flag + + + def _ensure_delete_infos(self, del_old_msg, del_new_msg): + ''' + check new version delete pkgs + ''' + del_new_pkg = {} + del_old_pkg = {} + ensure_delete_pkg = {} + for branch,del_new_msgs in del_new_msg.items(): + del_change_pkgs = [] + del_new_pkg[branch] = [new['name'] for new in del_new_msgs] + del_old_pkg[branch] = [old['name'] for old in del_old_msg[branch]] + delete_pkgs = list(set(del_new_pkg[branch]).difference(set(del_old_pkg[branch]))) + if delete_pkgs: + for del_new in del_new_msgs: + if del_new['name'] in delete_pkgs: + del_change_pkgs.append(del_new) + ensure_delete_pkg[branch] = del_change_pkgs + for branch,branch_del_pkgs in ensure_delete_pkg.items(): + if branch_del_pkgs: + log.info("branch:{},delete pkgs:{}".format(branch,branch_del_pkgs)) + return ensure_delete_pkg + + def _check_rpms_complete_and_repeat(self, old_msg, new_msg): + ''' + compare with old and new yaml msg, make sure package exist + ''' + old_pkg = {} + new_pkg = {} + same_pkg = {} + error_pkg = {} + change_infos = {} + error_pkg_flag = False + same_pkg_flag = False + log.info("rpms exists and repeat check") + for branch,new_msgs in new_msg.items(): + old_pkg[branch] = [] + new_pkg[branch] = [] + same_pkg[branch] = [] + change_pkgs = [] + old_msgs = old_msg[branch] + for old in old_msgs: + old_pkg[branch].append(old['name']) + for new in new_msgs: + if new['name'] in new_pkg[branch]: + same_pkg_flag = True + same_pkg[branch].append(new['name']) + new_pkg[branch].append(new['name']) + error_branch_pkgs = list(set(old_pkg[branch]).difference(set(new_pkg[branch]))) + if error_branch_pkgs: + error_pkg[branch] = error_branch_pkgs + error_pkg_flag = True + add_pkgs = list(set(new_pkg[branch]).difference(set(old_pkg[branch]))) + for new in new_msgs: + if new['name'] in add_pkgs: + change_pkgs.append(new) + change_infos[branch] = change_pkgs + for branch,pkgs in change_infos.items(): + if pkgs: + log.info("change in:{}".format(branch)) + for pkg in pkgs: + log.info(pkg) + if error_pkg_flag: + log.error("May be {0} should not be delete".format(error_pkg)) + raise SystemExit("ERROR: Please check your PR") + if same_pkg_flag: + log.error("The following {0} packages are duplicated in the YAML files".format(same_pkg)) + raise SystemExit("ERROR: Please check your PR") + return change_infos + + def _get_new_version_yaml_msg(self, yaml_path_list, manage_path,vtype='master'): + ''' + get new version yaml msg content + ''' + all_pack_msg = {} + all_del_msg = {} + for yaml_path in yaml_path_list: + result = {} + file_path = os.path.join(manage_path, yaml_path) + if vtype == 'master': + branch_infos = yaml_path.split('/') + branch = branch_infos[1] + if os.path.exists(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + yaml_packages = [pkg for pkg in result['packages']] + if 'delete' in branch_infos: + all_del_msg[branch] = yaml_packages + else: + if all_pack_msg.get(branch,''): + full_packags = all_pack_msg[branch] + yaml_packages + all_pack_msg[branch] = full_packags + else: + all_pack_msg[branch] = yaml_packages + else: + if 'delete' in branch_infos: + all_del_msg[branch] = [] + else: + all_pack_msg[branch] = [] + else: + branch_infos = yaml_path.split('/') + branch = branch_infos[0] + if 'delete' in branch_infos: + if os.path.exists(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + yaml_packages = [pkg for pkg in result['packages']] + all_del_msg[branch] = yaml_packages + else: + all_del_msg[branch] = [] + else: + if not all_pack_msg.get(branch,''): + all_branch_pkgs = self._get_complete_yaml_pkgs(branch) + if all_branch_pkgs: + all_pack_msg[branch] = all_branch_pkgs + return all_pack_msg,all_del_msg + + def _get_complete_yaml_pkgs(self,branch): + all_branch_pkgs = [] + if os.path.exists(os.path.join(self.manage_path, branch)): + standard_dirs = os.listdir(os.path.join(self.manage_path, branch)) + for standard_dir in standard_dirs: + file_path = os.path.join(self.manage_path, branch, standard_dir) + if not os.path.isdir(file_path) or standard_dir == 'delete': + standard_dirs.remove(standard_dir) + for c_dir in standard_dirs: + release_path = os.path.join(self.manage_path,branch, c_dir, 'pckg-mgmt.yaml') + if os.path.exists(release_path): + with open(release_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + all_branch_pkgs.extend(result['packages']) + return all_branch_pkgs + + + def check_pckg_yaml(self): """ check the obs_from branch_from in pckg-mgmt.yaml """ change = self._get_repo_change_file('openeuler', 'release-management', self.manage_path) - change_file = self._parse_commit_file(change) - self._check_yaml_format(change_file, self.manage_path) + change_file,master_change_file,new_version_change_file = self._parse_commit_file(change) + all_change_files = [*change_file, *master_change_file, *new_version_change_file] + self._check_yaml_format(all_change_files, self.manage_path) all_yaml_msg = self._get_allkey_msg(change_file, self.manage_path) change_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) - old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path, True) - self._check_rpms_integrity(old_yaml_msg, change_yaml_msg, change_file) - change_msg_list = self._get_diff_msg(old_yaml_msg, change_yaml_msg, change_file) - log.info(len(change_msg_list)) - self._ensure_delete_tags(change_msg_list, old_yaml_msg, all_yaml_msg) - self._check_key_in_yaml(change_msg_list, change_file) - error_flag1 = self._check_pkg_from(self.meta_path, change_msg_list, change_file, all_yaml_msg) - error_flag2 = self._check_date_time(change_msg_list, change_file) - error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) - error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) - if error_flag1 or error_flag2 or error_flag3 or error_flag4: - raise SystemExit("Please check your commit") + all_master_yaml_msg = self._get_allkey_msg(master_change_file, self.manage_path) + master_change_yaml_msg,del_master_change_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + new_version_change_msg,del_new_version_change_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') + self._rollback_get_msg(self.manage_path) + old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) + old_master_yaml_msg,del_old_master_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + old_new_version_msg,del_old_new_version_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') + add_infos = {} + if master_change_file: + log.info(master_change_file) + add_infos,move_infos= self._get_move_and_add(old_master_yaml_msg, master_change_yaml_msg) + add_flag = self._check_master_add_rules(add_infos, move_infos) + move_flag = self._check_master_move_rules(move_infos) + del_flag = self._check_master_del_rules(del_old_master_yaml_msg, del_master_change_yaml_msg) + date_flag = self._check_master_date_rules(add_infos) + if add_flag or move_flag or date_flag or del_flag: + raise SystemExit("Please check your commit") + if new_version_change_file: + log.info(new_version_change_file) + change_infos = self._check_rpms_complete_and_repeat(old_new_version_msg, new_version_change_msg) + change_delete_infos = self._ensure_delete_infos(del_old_new_version_msg, del_new_version_change_msg) + self._check_key_in_yaml_new(change_infos) + self._check_valid_release_branch(change_infos) + date_flag = self._check_pkg_date(change_infos) + correct_from, error_from = self._check_pkg_from_new(self.meta_path, change_infos) + error_flag_add = self._check_pkg_parent_from(change_infos, correct_from, error_from, add_infos) + error_flag_del = self._check_pkg_delete_new(self.meta_path, change_delete_infos) + if error_flag_add or error_flag_del or date_flag: + raise SystemExit("Please check your commit") + if change_file: + log.info(change_file) + self._check_rpms_integrity(old_yaml_msg, change_yaml_msg, change_file) + change_msg_list = self._get_diff_msg(old_yaml_msg, change_yaml_msg, change_file) + self._ensure_delete_tags(change_msg_list, old_yaml_msg, all_yaml_msg) + self._check_key_in_yaml(change_msg_list, change_file) + error_flag1 = self._check_pkg_from(self.meta_path, change_msg_list, change_file, all_yaml_msg) + error_flag2 = self._check_date_time(change_msg_list, change_file) + error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) + error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) + if error_flag1 or error_flag2 or error_flag3 or error_flag4: + raise SystemExit("Please check your commit") if __name__ == "__main__": kw = {"branch":"master", - "gitee_user":"", - "gitee_pwd":"", - "pr_id":"", - "obs_meta_path":"***", - "release_management_path":"***"} + "gitee_user":"dongjie110", + "gitee_pwd":"Dj@6445240", + "pr_id":"130", + "obs_meta_path":"/home/0712/obs_meta", + "release_management_path":"/home/0712/release-management"} check = CheckReleaseManagement(**kw) - check.check_pckg_yaml() + check.check_pckg_yaml() \ No newline at end of file diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index ebf73f2..ec8a193 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -1,335 +1,596 @@ -#!/bin/env python3 -# -*- encoding=utf8 -*- -#****************************************************************************** -# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. -# licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Author: wangchong -# Create: 2021-06-08 -# ****************************************************************************** -""" -Synchronize the obs_meta file according to the pckg-mgmt.yaml file -""" - -import os -import sys -import yaml -import shutil -now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) -sys.path.append(os.path.join(now_path, "..")) -from common.log_obs import log -from common.parser_config import ParserConfigIni - - -class SyncPckgMgmt(object): - """ - keep pckg-mgmt.yaml and obs_meta in sync - """ - def __init__(self, **kwargs): - """ - giteeuser: gitee user name - giteeuserpwd: gitee password - """ - self.kwargs = kwargs - self.giteeuser = self.kwargs['gitee_user'] - self.giteeuserpwd = self.kwargs['gitee_pwd'] - self.obs_meta_path = self.kwargs['obs_meta_path'] - self.release_management_path = self.kwargs['release_management_path'] - par = ParserConfigIni() - self.obs_ignored_package = par.get_obs_ignored_package() - - def _get_change_file(self): - """ - get release-managemnet change file - """ - if os.path.exists(self.release_management_path): - os.chdir(self.release_management_path) - cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" - result = os.popen(cmd).read().split('\n') - change_file = [x for x in result if x != ''] - return change_file - else: - log.error("%s not exist!" % self.release_management_path) - sys.exit(1) - - def _get_yaml_file_msg(self, file_path): - """ - get pckg-mgmt.yaml file msg - """ - file_msg = None - if os.path.exists(file_path): - with open(file_path, "r", encoding='utf-8') as f: - file_msg = yaml.load(f, Loader=yaml.FullLoader) - return file_msg - - def _check_pckg_yaml(self, file_msg, branch): - """ - check pckg-mgmt.yaml file - """ - proj = branch.replace("-", ":") - flag = False - for title in file_msg['packages']: - for msg in file_msg['packages'][title]: - if msg['branch_to'] != branch: - flag = True - log.error("%s branch_to is error, please check yaml." % msg['name']) - if not msg['obs_to'].startswith(proj): - flag = True - log.error("%s obs_to is error, please check yaml." % msg['name']) - return flag - - def _parse_yaml_msg(self, file_msg, yaml_type): - """ - parse yaml file msg - """ - tmp = {} - msg = [] - del_msg = [] - prj_pkg = {} - if yaml_type == "old": - for pckg in file_msg['packages']['natural']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - else: - for label in file_msg['packages']['everything']: - for pckg in file_msg['packages']['everything'][label]: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - for pckg in file_msg['packages']['epol']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) - return msg, del_msg, prj_pkg - - def _write_prj_meta_file(self, file_path, proj): - """ - write project meta file - """ - if "Epol" in proj: - main_proj = proj.replace(':Epol', '') - con_proj = main_proj.replace(':', '_').lower() - epol_repo_aarch64 = "\n " % ( - main_proj, con_proj) - epol_repo_x86 = "\n " % ( - main_proj, con_proj) - else: - main_proj = proj - con_proj = main_proj.replace(':', '_').lower() - epol_repo_aarch64 = "" - epol_repo_x86 = "" - file_msg = """ - - <description/> - <person userid="Admin" role="maintainer"/> - <useforbuild> - <disable/> - </useforbuild> - <repository name="standard_aarch64"> - <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_aarch64"/>{3} - <arch>aarch64</arch> - </repository> - <repository name="standard_x86_64"> - <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_x86_64"/>{4} - <arch>x86_64</arch> - </repository> -</project> -""".format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) - f = open(file_path, "w") - f.write(file_msg) - f.close() - - def _write_selfbuild_meta_file(self, file_path, proj): - """ - write selfbuild project meta file - """ - file_msg = """<project name="{0}:selfbuild:BaseOS"> - <title/> - <description/> - <person userid="Admin" role="maintainer"/> - <repository name="{1}_standard_aarch64"> - <arch>aarch64</arch> - </repository> - <repository name="{1}_standard_x86_64"> - <arch>x86_64</arch> - </repository> - <repository name="{1}_epol_aarch64"> - <arch>aarch64</arch> - </repository> - <repository name="{1}_epol_x86_64"> - <arch>x86_64</arch> - </repository> -</project> -""".format(proj, proj.lower().replace(':', '_')) - f = open(file_path, "w") - f.write(file_msg) - f.close() - - def _write_service_file(self, filepath, package, pkg_branch): - ''' - write service file - ''' - file_msg = """<services> - <service name="tar_scm_kernel_repo"> - <param name="scm">repo</param> - <param name="url">next/{}/{}</param> - </service> -</services>""".format(pkg_branch,package) - try: - with open(os.path.join(filepath,'_service'),'w') as f: - f.write(file_msg) - except Exception as e: - print (e) - - def _add_pkg_service(self, tmp): - """ - add obs_meta packages _service file - """ - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if tmp['branch_from'] == "master" and tmp['pkgname'] not in self.obs_ignored_package: - self._write_service_file(pkg_path, tmp['pkgname'], tmp['branch_to']) - if os.path.exists(pkg_service_path): - log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - if tmp['branch_from'] == "master": - branch = "openEuler" - else: - branch = tmp['branch_from'] - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) - if os.system(cmd) == 0: - log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.error("copy %s service file failed!" % tmp['pkgname']) - - def _add_prj_meta_pkgs_service(self, msg): - for tmp in msg: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to']: - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - self._add_pkg_service(tmp) - - def _del_pkg(self, tmp): - """ - delete obs_meta packages - """ - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - if os.path.exists(pkg_path): - shutil.rmtree(pkg_path) - log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - - def _verify_meta_file(self, prj_pkg): - """ - verify obs_meta with pckg-mgmt.yaml - """ - for proj, pkg in prj_pkg.items(): - if proj != "branch": - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) - need_del_pkg = set(meta_pkglist) - set(pkg) - log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) - for del_pkg in need_del_pkg: - tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} - self._del_pkg(tmp) - - def _push_code(self): - """ - push code to gitee repo - """ - if os.path.exists(self.obs_meta_path): - os.chdir(self.obs_meta_path) - cmd = "git status -s" - if os.popen(cmd).read(): - cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" - if os.system(cmd) == 0: - cmd = "git push -f" - for i in range(5): - if os.system(cmd) == 0: - log.info("push code to gitee repo succeed!") - return 0 - else: - log.error("push code failed, try again...") - raise SystemExit("push code to gitee repo Failed!") - else: - log.info("No change, nothing to commit!") - return "nothing to push" - else: - log.error("%s not exist!" % self.obs_meta_path) - return -1 - - def sync_yaml_meta(self): - """ - integration of functions - """ - change_file = self._get_change_file() - yaml_dict = {} - for line in change_file: - log.info("line:%s" % line) - name = list(line.split())[1] - branch = name.split('/')[0] - file_path = os.path.join(self.release_management_path, name) - yaml_dict = self._get_yaml_file_msg(file_path) - if not yaml_dict: - log.info("%s file content is empty!" % name) - else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg) - else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg) - for tmp in del_msg: - self._del_pkg(tmp) - self._verify_meta_file(prj_pkg) - ret = self._push_code() - return ret - - -if __name__ == "__main__": - kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} - mgmt = SyncPckgMgmt(**kw) - mgmt.sync_yaml_meta() +#!/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: wangchong +# Create: 2021-06-08 +# ****************************************************************************** +""" +Synchronize the obs_meta file according to the pckg-mgmt.yaml file +""" + +import os +import re +import sys +import yaml +import shutil +import datetime +now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) +sys.path.append(os.path.join(now_path, "..")) +from common.log_obs import log + + +class SyncPckgMgmt(object): + """ + keep pckg-mgmt.yaml and obs_meta in sync + """ + def __init__(self, **kwargs): + """ + giteeuser: gitee user name + giteeuserpwd: gitee password + """ + self.kwargs = kwargs + self.giteeuser = self.kwargs['gitee_user'] + self.giteeuserpwd = self.kwargs['gitee_pwd'] + self.obs_meta_path = self.kwargs['obs_meta_path'] + self.release_management_path = self.kwargs['release_management_path'] + + def _get_change_file(self): + """ + get release-managemnet change file + """ + if os.path.exists(self.release_management_path): + os.chdir(self.release_management_path) + cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" + result = os.popen(cmd).read().split('\n') + change_file = [x for x in result if x != ''] + return change_file + else: + log.error("%s not exist!" % self.release_management_path) + sys.exit(1) + + def _get_yaml_file_msg(self, file_path): + """ + get pckg-mgmt.yaml file msg + """ + file_msg = None + if os.path.exists(file_path): + with open(file_path, "r", encoding='utf-8') as f: + file_msg = yaml.load(f, Loader=yaml.FullLoader) + return file_msg + + def _check_pckg_yaml(self, file_msg, branch): + """ + check pckg-mgmt.yaml file + """ + proj = branch.replace("-", ":") + flag = False + for title in file_msg['packages']: + for msg in file_msg['packages'][title]: + if msg['branch_to'] != branch: + flag = True + log.error("%s branch_to is error, please check yaml." % msg['name']) + if not msg['obs_to'].startswith(proj): + flag = True + log.error("%s obs_to is error, please check yaml." % msg['name']) + return flag + + def _parse_yaml_msg(self, file_msg, yaml_type): + """ + parse yaml file msg + """ + tmp = {} + msg = [] + del_msg = [] + prj_pkg = {} + if yaml_type == "old": + for pckg in file_msg['packages']['natural']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) + elif yaml_type == 'multi-new': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['source_dir'], + 'branch_to': pckg['destination_dir'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['destination_dir'] + msg.append(tmp) + elif yaml_type == 'multi-new-delete': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name']} + msg.append(tmp) + else: + for label in file_msg['packages']['everything']: + for pckg in file_msg['packages']['everything'][label]: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + for pckg in file_msg['packages']['epol']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + return msg, del_msg, prj_pkg + + def _write_prj_meta_file(self, file_path, proj): + """ + write project meta file + """ + if "Epol" in proj: + main_proj = proj.replace(':Epol', '') + con_proj = main_proj.replace(':', '_').lower() + epol_repo_aarch64 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_aarch64\"/>" % ( + main_proj, con_proj) + epol_repo_x86 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_x86_64\"/>" % ( + main_proj, con_proj) + else: + main_proj = proj + con_proj = main_proj.replace(':', '_').lower() + epol_repo_aarch64 = "" + epol_repo_x86 = "" + # if 'openEuler:Testing' in proj:#Testing + if 'master' in file_path: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="standard_aarch64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <useforbuild> + <disable/> + </useforbuild> + <repository name="standard_aarch64"> + <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_aarch64"/>{3} + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64"> + <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_x86_64"/>{4} + <arch>x86_64</arch> + </repository> +</project> + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + f = open(file_path, "w") + f.write(file_msg) + f.close() + + def _write_selfbuild_meta_file(self, file_path, proj): + """ + write selfbuild project meta file + """ + file_msg = """<project name="{0}:selfbuild:BaseOS"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="{1}_standard_aarch64"> + <arch>aarch64</arch> + </repository> + <repository name="{1}_standard_x86_64"> + <arch>x86_64</arch> + </repository> + <repository name="{1}_epol_aarch64"> + <arch>aarch64</arch> + </repository> + <repository name="{1}_epol_x86_64"> + <arch>x86_64</arch> + </repository> +</project> +""".format(proj, proj.lower().replace(':', '_')) + f = open(file_path, "w") + f.write(file_msg) + f.close() + + def _add_pkg_service(self, tmp): + """ + add obs_meta packages _service file + """ + success_pkg_name = '' + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + return success_pkg_name + + def _move_pkg_service(self,tmp): + success_pkg_name = '' + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + if not os.path.exists(mv_to_path): + os.makedirs(mv_to_path) + if not os.path.exists(pkg_path): + cmd = "mv %s %s" % (from_pkg_path,mv_to_path) + if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] + log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return success_pkg_name + + def _add_master_pkg_service(self,package): + """ + create and add master service to repo obs_meta + """ + filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) + if not os.path.exists(filepath): + os.makedirs(filepath) + file_msg = """<services> + <service name="tar_scm"> + <param name="scm">git</param> + <param name="url">git@gitee.com:src-openeuler/{}.git</param> + <param name="exclude">*</param> + <param name="extract">*</param> + <param name="revision">master</param> + </service> +</services>""".format(package) + try: + with open(os.path.join(filepath,'_service'),'w') as f: + f.write(file_msg) + log.info("add %s _service success!" % (filepath)) + except Exception as e: + print (e) + log.info("add %s _service failed!" % (filepath)) + + def _move_master_pkg_service(self, msg): + """ + copy branch master obs_meta packages _service file + """ + change_pkgs = [] + for tmp in msg: + if tmp['obs_to'] == 'openEuler:Factory': + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if not os.path.exists(pkg_path): + self._add_master_pkg_service(tmp['pkgname']) + change_pkgs.append(tmp['pkgname']) + else: + pkg_name = self._move_pkg_service(tmp) + change_pkgs.append(pkg_name) + # from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + # pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + # pkg_service_path = os.path.join(pkg_path, "_service") + # if not os.path.exists(pkg_path): + # os.makedirs(pkg_path) + # if not os.path.exists(pkg_service_path): + # cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + # if os.system(cmd) == 0: + # change_pkgs.append(tmp['pkgname']) + # log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + # else: + # log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return change_pkgs + + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): + pkg_names = [] + for tmp in msg: + if 'multi_version' not in branch_infos: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'master' not in branch_infos: + if tmp['branch_from'] == tmp['branch_to'] and tmp['obs_from'] != tmp['obs_to']: + pkg_name = self._move_pkg_service(tmp) + else: + pkg_name = self._add_pkg_service(tmp) + if pkg_name: + pkg_names.append(pkg_name) + return pkg_names + + + def _del_pkg(self, tmp): + """ + delete obs_meta packages + """ + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] + + def _del_pkg_new(self, pkgs, branch): + """ + find in branch dirs and delete pkg + """ + need_del_path = [] + correct_del_pkgs = [] + release_changes = {} + branch_dir = os.path.join(self.obs_meta_path,branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path and 'bringInRely' not in c_path and 'RISC-V' not in c_path: + correct_del_pkgs.append(name) + need_del_path.append(c_path) + for pkg_path in need_del_path: + if os.path.exists(pkg_path): + pkg_path_infos = pkg_path.split('/') + shutil.rmtree(pkg_path) + log.info("delete %s %s succeed!" % (branch,pkg_path_infos[-1])) + release_branch = pkg_path_infos[-3] + if release_branch == 'master': + obs_project = pkg_path_infos[-2].replace(":","-") + release_path = os.path.join(self.release_management_path,release_branch,obs_project) + if release_changes.get(release_path): + release_changes[release_path].append(pkg_path_infos[-1]) + else: + release_changes[release_path] = [pkg_path_infos[-1]] + else: + standard_dirs = ['epol','everything-exclude-baseos','baseos'] + for c_dir in standard_dirs: + release_path = os.path.join(self.release_management_path,release_branch,c_dir) + if release_changes.get(release_path): + release_changes[release_path].append(pkg_path_infos[-1]) + else: + release_changes[release_path] = [pkg_path_infos[-1]] + return correct_del_pkgs,release_changes + + def _del_pckg_from_yaml(self, release_changes): + if release_changes: + for pkgs_path,pkgs in release_changes.items(): + result = {} + change_flag = False + complete_path = os.path.join(pkgs_path,'pckg-mgmt.yaml') + old_yaml = self._get_yaml_file_msg(complete_path) + pkgs_list = old_yaml['packages'] + for i in range(len(pkgs_list)-1,-1,-1): + if pkgs_list[i]['name'] in pkgs: + pkgs_list.pop(i) + change_flag = True + if change_flag: + result['packages'] = pkgs_list + with open(complete_path, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + + + def _verify_meta_file(self, prj_pkg): + """ + verify obs_meta with pckg-mgmt.yaml + """ + for proj, pkg in prj_pkg.items(): + if proj != "branch": + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + if len(list(prj_pkg.keys())) >= 3: + need_del_pkg = [] + else: + need_del_pkg = set(meta_pkglist) - set(pkg) + log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) + if need_del_pkg: + for del_pkg in need_del_pkg: + tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} + self._del_pkg(tmp) + return list(need_del_pkg) + + + def _push_code(self,repo): + """ + push code to gitee repo + """ + if os.path.exists(repo): + os.chdir(repo) + cmd = "git status -s" + if os.popen(cmd).read(): + cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" + if os.system(cmd) == 0: + cmd = "git push -f" + for i in range(5): + if os.system(cmd) == 0: + log.info("push code to gitee repo succeed!") + return 0 + else: + log.error("push code failed, try again...") + raise SystemExit("push code to gitee repo Failed!") + else: + log.info("No change, nothing to commit!") + return "nothing to push" + else: + log.error("%s not exist!" % repo) + return -1 + + def _write_release_yaml(self,change_pkgs,branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs and len(change_pkgs) < 50: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + else: + pull_request = commitid + log.info("ignore write release change yaml file success") + + + def sync_yaml_meta(self): + """ + integration of functions + """ + change_file = self._get_change_file() + master_change_file = [] + yaml_dict = {} + master_prj_pkg = [] + for line in change_file: + log.info("line:%s" % line) + name = list(line.split())[1] + file_path = os.path.join(self.release_management_path, name) + yaml_dict = self._get_yaml_file_msg(file_path) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = branch_infos[1] + if branch != 'delete': + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + # master_prj_pkg.append(prj_pkg) + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + move_change_pkgs = self._move_master_pkg_service(msg) + self._write_release_yaml(move_change_pkgs,branch_infos[0]) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") + pkgs = [tmp['pkgname'] for tmp in msg] + del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs,'{}'.format(branch_infos[0])) + self._write_release_yaml(del_change_pkgs,branch_infos[0]) + self._del_pckg_from_yaml(yaml_changes) + elif 'multi_version' in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = pkg_names + del_change_pkgs + complete_path = os.path.join(branch_infos[0],branch_infos[1]) + self._write_release_yaml(change_pkgs,complete_path) + else: + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + if 'delete' not in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + self._write_release_yaml(pkg_names,branch_infos[0]) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") + pkgs = [tmp['pkgname'] for tmp in msg] + del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs,branch_infos[0]) + self._write_release_yaml(del_change_pkgs,branch_infos[0]) + self._del_pckg_from_yaml(yaml_changes) + else: + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + for tmp in del_msg: + self._del_pkg(tmp) + del_change_pkgs = self._verify_meta_file(prj_pkg) + # if master_prj_pkg: + # for prj_pkg in master_prj_pkg: + # master_del_pkgs = self._verify_meta_file(prj_pkg) + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) + return ret + + +if __name__ == "__main__": + # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} + mgmt = SyncPckgMgmt(**kw) + mgmt.sync_yaml_meta() \ No newline at end of file -- Gitee From d859383c0b108673f665bbd10bd564d5c9edf2e8 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 20 Jul 2022 10:06:01 +0800 Subject: [PATCH 02/19] sync code --- core/check_release_management.py | 17 +- core/sync_pckg_mgmt.py | 1210 +++++++++++++++--------------- 2 files changed, 622 insertions(+), 605 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 589da52..77143b0 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -484,7 +484,7 @@ class CheckReleaseManagement(object): else: branch = branch.replace("-",":") from_result = self._find_master_meta_path(pkg) - move_result = self._find_in_delete_pkgs(branch,pkg,move_infos) + move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) if pkg['obs_to'] != branch: error_flag = True log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) @@ -502,7 +502,7 @@ class CheckReleaseManagement(object): log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag - def _find_in_delete_pkgs(self,branch,pkg,move_infos): + def _find_in_delete_pkgs(self, branch, pkg, move_infos): ''' check move pkg in from branch yaml is deleted ''' @@ -692,8 +692,6 @@ class CheckReleaseManagement(object): ''' re-check error pkg_from and find in parent branch add ''' - print (correct_from) - print (error_from) error_flag = False branch_msg_path = os.path.join(self.manage_path, "valid_release_branches.yaml") with open(branch_msg_path, 'r', encoding='utf-8') as f: @@ -703,7 +701,6 @@ class CheckReleaseManagement(object): for master_branch, master_pkgs in add_infos.items(): master_pkgnames = [line['name'] for line in master_pkgs] all_master_add[master_branch] = master_pkgnames - print (all_master_add) if error_from: for branch, pkgs in error_from.items(): error_names = [pkg['name'] for pkg in pkgs] @@ -981,10 +978,10 @@ class CheckReleaseManagement(object): if __name__ == "__main__": kw = {"branch":"master", - "gitee_user":"dongjie110", - "gitee_pwd":"Dj@6445240", - "pr_id":"130", - "obs_meta_path":"/home/0712/obs_meta", - "release_management_path":"/home/0712/release-management"} + "gitee_user":"", + "gitee_pwd":"", + "pr_id":"", + "obs_meta_path":"***", + "release_management_path":"***"} check = CheckReleaseManagement(**kw) check.check_pckg_yaml() \ No newline at end of file diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index ec8a193..c333c8a 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -1,596 +1,616 @@ -#!/bin/env python3 -# -*- encoding=utf8 -*- -#****************************************************************************** -# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. -# licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# Author: wangchong -# Create: 2021-06-08 -# ****************************************************************************** -""" -Synchronize the obs_meta file according to the pckg-mgmt.yaml file -""" - -import os -import re -import sys -import yaml -import shutil -import datetime -now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) -sys.path.append(os.path.join(now_path, "..")) -from common.log_obs import log - - -class SyncPckgMgmt(object): - """ - keep pckg-mgmt.yaml and obs_meta in sync - """ - def __init__(self, **kwargs): - """ - giteeuser: gitee user name - giteeuserpwd: gitee password - """ - self.kwargs = kwargs - self.giteeuser = self.kwargs['gitee_user'] - self.giteeuserpwd = self.kwargs['gitee_pwd'] - self.obs_meta_path = self.kwargs['obs_meta_path'] - self.release_management_path = self.kwargs['release_management_path'] - - def _get_change_file(self): - """ - get release-managemnet change file - """ - if os.path.exists(self.release_management_path): - os.chdir(self.release_management_path) - cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" - result = os.popen(cmd).read().split('\n') - change_file = [x for x in result if x != ''] - return change_file - else: - log.error("%s not exist!" % self.release_management_path) - sys.exit(1) - - def _get_yaml_file_msg(self, file_path): - """ - get pckg-mgmt.yaml file msg - """ - file_msg = None - if os.path.exists(file_path): - with open(file_path, "r", encoding='utf-8') as f: - file_msg = yaml.load(f, Loader=yaml.FullLoader) - return file_msg - - def _check_pckg_yaml(self, file_msg, branch): - """ - check pckg-mgmt.yaml file - """ - proj = branch.replace("-", ":") - flag = False - for title in file_msg['packages']: - for msg in file_msg['packages'][title]: - if msg['branch_to'] != branch: - flag = True - log.error("%s branch_to is error, please check yaml." % msg['name']) - if not msg['obs_to'].startswith(proj): - flag = True - log.error("%s obs_to is error, please check yaml." % msg['name']) - return flag - - def _parse_yaml_msg(self, file_msg, yaml_type): - """ - parse yaml file msg - """ - tmp = {} - msg = [] - del_msg = [] - prj_pkg = {} - if yaml_type == "old": - for pckg in file_msg['packages']['natural']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) - elif yaml_type == 'master': - for pckg in file_msg['packages']: - tmp = {'pkgname': pckg['name'], 'branch_from': 'master', - 'branch_to': 'master', 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = 'master' - msg.append(tmp) - elif yaml_type == 'multi-new': - for pckg in file_msg['packages']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['source_dir'], - 'branch_to': pckg['destination_dir'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['destination_dir'] - msg.append(tmp) - elif yaml_type == 'multi-new-delete': - for pckg in file_msg['packages']: - tmp = {'pkgname': pckg['name']} - msg.append(tmp) - else: - for label in file_msg['packages']['everything']: - for pckg in file_msg['packages']['everything'][label]: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - for pckg in file_msg['packages']['epol']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] - msg.append(tmp) - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) - return msg, del_msg, prj_pkg - - def _write_prj_meta_file(self, file_path, proj): - """ - write project meta file - """ - if "Epol" in proj: - main_proj = proj.replace(':Epol', '') - con_proj = main_proj.replace(':', '_').lower() - epol_repo_aarch64 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_aarch64\"/>" % ( - main_proj, con_proj) - epol_repo_x86 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_x86_64\"/>" % ( - main_proj, con_proj) - else: - main_proj = proj - con_proj = main_proj.replace(':', '_').lower() - epol_repo_aarch64 = "" - epol_repo_x86 = "" - # if 'openEuler:Testing' in proj:#Testing - if 'master' in file_path: - file_msg = """<project name="{0}"> - <title/> - <description/> - <person userid="Admin" role="maintainer"/> - <repository name="standard_aarch64" rebuild="direct"> - <path project="openEuler:selfbuild:BaseOS" repository="baseos_aarch64"/> - <arch>aarch64</arch> - </repository> - <repository name="standard_x86_64" rebuild="direct"> - <path project="openEuler:selfbuild:BaseOS" repository="baseos_x86_64"/> - <arch>x86_64</arch> - </repository> -</project> - """.format(proj) - else: - file_msg = """<project name="{0}"> - <title/> - <description/> - <person userid="Admin" role="maintainer"/> - <useforbuild> - <disable/> - </useforbuild> - <repository name="standard_aarch64"> - <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_aarch64"/>{3} - <arch>aarch64</arch> - </repository> - <repository name="standard_x86_64"> - <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_x86_64"/>{4} - <arch>x86_64</arch> - </repository> -</project> - """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) - f = open(file_path, "w") - f.write(file_msg) - f.close() - - def _write_selfbuild_meta_file(self, file_path, proj): - """ - write selfbuild project meta file - """ - file_msg = """<project name="{0}:selfbuild:BaseOS"> - <title/> - <description/> - <person userid="Admin" role="maintainer"/> - <repository name="{1}_standard_aarch64"> - <arch>aarch64</arch> - </repository> - <repository name="{1}_standard_x86_64"> - <arch>x86_64</arch> - </repository> - <repository name="{1}_epol_aarch64"> - <arch>aarch64</arch> - </repository> - <repository name="{1}_epol_x86_64"> - <arch>x86_64</arch> - </repository> -</project> -""".format(proj, proj.lower().replace(':', '_')) - f = open(file_path, "w") - f.write(file_msg) - f.close() - - def _add_pkg_service(self, tmp): - """ - add obs_meta packages _service file - """ - success_pkg_name = '' - if 'Multi-Version' in tmp['branch_from']: - dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) - from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) - pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if tmp['branch_from'] == "master": - branch = "openEuler" - else: - branch = tmp['branch_from'] - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) - if os.system(cmd) == 0: - success_pkg_name = tmp['pkgname'] - log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.error("copy %s service file failed!" % tmp['pkgname']) - return success_pkg_name - - def _move_pkg_service(self,tmp): - success_pkg_name = '' - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) - if not os.path.exists(mv_to_path): - os.makedirs(mv_to_path) - if not os.path.exists(pkg_path): - cmd = "mv %s %s" % (from_pkg_path,mv_to_path) - if os.system(cmd) == 0: - success_pkg_name = tmp['pkgname'] - log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - else: - log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - return success_pkg_name - - def _add_master_pkg_service(self,package): - """ - create and add master service to repo obs_meta - """ - filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) - if not os.path.exists(filepath): - os.makedirs(filepath) - file_msg = """<services> - <service name="tar_scm"> - <param name="scm">git</param> - <param name="url">git@gitee.com:src-openeuler/{}.git</param> - <param name="exclude">*</param> - <param name="extract">*</param> - <param name="revision">master</param> - </service> -</services>""".format(package) - try: - with open(os.path.join(filepath,'_service'),'w') as f: - f.write(file_msg) - log.info("add %s _service success!" % (filepath)) - except Exception as e: - print (e) - log.info("add %s _service failed!" % (filepath)) - - def _move_master_pkg_service(self, msg): - """ - copy branch master obs_meta packages _service file - """ - change_pkgs = [] - for tmp in msg: - if tmp['obs_to'] == 'openEuler:Factory': - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - if not os.path.exists(pkg_path): - self._add_master_pkg_service(tmp['pkgname']) - change_pkgs.append(tmp['pkgname']) - else: - pkg_name = self._move_pkg_service(tmp) - change_pkgs.append(pkg_name) - # from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - # pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - # pkg_service_path = os.path.join(pkg_path, "_service") - # if not os.path.exists(pkg_path): - # os.makedirs(pkg_path) - # if not os.path.exists(pkg_service_path): - # cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - # if os.system(cmd) == 0: - # change_pkgs.append(tmp['pkgname']) - # log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - # else: - # log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - return change_pkgs - - - def _add_prj_meta_pkgs_service(self, msg, branch_infos): - pkg_names = [] - for tmp in msg: - if 'multi_version' not in branch_infos: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - if 'master' not in branch_infos: - if tmp['branch_from'] == tmp['branch_to'] and tmp['obs_from'] != tmp['obs_to']: - pkg_name = self._move_pkg_service(tmp) - else: - pkg_name = self._add_pkg_service(tmp) - if pkg_name: - pkg_names.append(pkg_name) - return pkg_names - - - def _del_pkg(self, tmp): - """ - delete obs_meta packages - """ - if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) - pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) - else: - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - if os.path.exists(pkg_path): - shutil.rmtree(pkg_path) - log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - return tmp['pkgname'] - - def _del_pkg_new(self, pkgs, branch): - """ - find in branch dirs and delete pkg - """ - need_del_path = [] - correct_del_pkgs = [] - release_changes = {} - branch_dir = os.path.join(self.obs_meta_path,branch) - for root, dirs, files in os.walk(branch_dir, True): - for name in dirs: - c_path = os.path.join(root, name) - if name in pkgs and 'Bak' not in c_path and 'bringInRely' not in c_path and 'RISC-V' not in c_path: - correct_del_pkgs.append(name) - need_del_path.append(c_path) - for pkg_path in need_del_path: - if os.path.exists(pkg_path): - pkg_path_infos = pkg_path.split('/') - shutil.rmtree(pkg_path) - log.info("delete %s %s succeed!" % (branch,pkg_path_infos[-1])) - release_branch = pkg_path_infos[-3] - if release_branch == 'master': - obs_project = pkg_path_infos[-2].replace(":","-") - release_path = os.path.join(self.release_management_path,release_branch,obs_project) - if release_changes.get(release_path): - release_changes[release_path].append(pkg_path_infos[-1]) - else: - release_changes[release_path] = [pkg_path_infos[-1]] - else: - standard_dirs = ['epol','everything-exclude-baseos','baseos'] - for c_dir in standard_dirs: - release_path = os.path.join(self.release_management_path,release_branch,c_dir) - if release_changes.get(release_path): - release_changes[release_path].append(pkg_path_infos[-1]) - else: - release_changes[release_path] = [pkg_path_infos[-1]] - return correct_del_pkgs,release_changes - - def _del_pckg_from_yaml(self, release_changes): - if release_changes: - for pkgs_path,pkgs in release_changes.items(): - result = {} - change_flag = False - complete_path = os.path.join(pkgs_path,'pckg-mgmt.yaml') - old_yaml = self._get_yaml_file_msg(complete_path) - pkgs_list = old_yaml['packages'] - for i in range(len(pkgs_list)-1,-1,-1): - if pkgs_list[i]['name'] in pkgs: - pkgs_list.pop(i) - change_flag = True - if change_flag: - result['packages'] = pkgs_list - with open(complete_path, "w", encoding='utf-8') as f: - yaml.dump(result, f, default_flow_style=False, sort_keys=False) - - - def _verify_meta_file(self, prj_pkg): - """ - verify obs_meta with pckg-mgmt.yaml - """ - for proj, pkg in prj_pkg.items(): - if proj != "branch": - if 'Multi-Version' in prj_pkg['branch']: - dir_name = '{}/{}'.format(prj_pkg['branch'],proj) - meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) - meta_pkglist = os.listdir(meta_path) - else: - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) - if 'README.md' in meta_pkglist: - meta_pkglist.remove('README.md') - if len(list(prj_pkg.keys())) >= 3: - need_del_pkg = [] - else: - need_del_pkg = set(meta_pkglist) - set(pkg) - log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) - if need_del_pkg: - for del_pkg in need_del_pkg: - tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} - self._del_pkg(tmp) - return list(need_del_pkg) - - - def _push_code(self,repo): - """ - push code to gitee repo - """ - if os.path.exists(repo): - os.chdir(repo) - cmd = "git status -s" - if os.popen(cmd).read(): - cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" - if os.system(cmd) == 0: - cmd = "git push -f" - for i in range(5): - if os.system(cmd) == 0: - log.info("push code to gitee repo succeed!") - return 0 - else: - log.error("push code failed, try again...") - raise SystemExit("push code to gitee repo Failed!") - else: - log.info("No change, nothing to commit!") - return "nothing to push" - else: - log.error("%s not exist!" % repo) - return -1 - - def _write_release_yaml(self,change_pkgs,branch): - ''' - write change info to release_change.yaml - ''' - if change_pkgs and len(change_pkgs) < 50: - change_str = " ".join(change_pkgs) - os.chdir(self.release_management_path) - commit_cmd = 'git rev-parse HEAD' - commitid = os.popen(commit_cmd).read().split('\n')[0] - content_cmd = "git log --oneline -1" - content = os.popen(content_cmd).read().split('\n')[0] - reg=re.compile(r"(?<=!)\d+") - match=reg.search(content) - if match: - pr_id = match.group(0) - pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) - datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') - release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") - with open(release_change_yaml) as file: - result = yaml.load(file, Loader=yaml.FullLoader) - change_dic = { - 'pr':pull_request, - 'description':content, - 'changed_packages':change_str, - 'date':datestr - } - result['release-change'].append(change_dic) - with open(release_change_yaml, "w", encoding='utf-8') as f: - yaml.dump(result, f, default_flow_style=False, sort_keys=False) - log.info("write release change yaml file success") - else: - pull_request = commitid - log.info("ignore write release change yaml file success") - - - def sync_yaml_meta(self): - """ - integration of functions - """ - change_file = self._get_change_file() - master_change_file = [] - yaml_dict = {} - master_prj_pkg = [] - for line in change_file: - log.info("line:%s" % line) - name = list(line.split())[1] - file_path = os.path.join(self.release_management_path, name) - yaml_dict = self._get_yaml_file_msg(file_path) - branch_infos = name.split('/') - if 'master' in branch_infos: - branch = branch_infos[1] - if branch != 'delete': - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") - # master_prj_pkg.append(prj_pkg) - pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) - move_change_pkgs = self._move_master_pkg_service(msg) - self._write_release_yaml(move_change_pkgs,branch_infos[0]) - else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") - pkgs = [tmp['pkgname'] for tmp in msg] - del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs,'{}'.format(branch_infos[0])) - self._write_release_yaml(del_change_pkgs,branch_infos[0]) - self._del_pckg_from_yaml(yaml_changes) - elif 'multi_version' in branch_infos: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") - pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) - del_change_pkgs = self._verify_meta_file(prj_pkg) - change_pkgs = pkg_names + del_change_pkgs - complete_path = os.path.join(branch_infos[0],branch_infos[1]) - self._write_release_yaml(change_pkgs,complete_path) - else: - if not yaml_dict: - log.info("%s file content is empty!" % name) - elif isinstance(yaml_dict['packages'], list): - if 'delete' not in branch_infos: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") - pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) - self._write_release_yaml(pkg_names,branch_infos[0]) - else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") - pkgs = [tmp['pkgname'] for tmp in msg] - del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs,branch_infos[0]) - self._write_release_yaml(del_change_pkgs,branch_infos[0]) - self._del_pckg_from_yaml(yaml_changes) - else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) - else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) - for tmp in del_msg: - self._del_pkg(tmp) - del_change_pkgs = self._verify_meta_file(prj_pkg) - # if master_prj_pkg: - # for prj_pkg in master_prj_pkg: - # master_del_pkgs = self._verify_meta_file(prj_pkg) - ret = self._push_code(self.obs_meta_path) - self._push_code(self.release_management_path) - return ret - - -if __name__ == "__main__": - # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} - kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} - mgmt = SyncPckgMgmt(**kw) +#!/bin/env python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: wangchong +# Create: 2021-06-08 +# ****************************************************************************** +""" +Synchronize the obs_meta file according to the pckg-mgmt.yaml file +""" + +import os +import re +import sys +import yaml +import shutil +import datetime +now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) +sys.path.append(os.path.join(now_path, "..")) +from common.log_obs import log +from common.parser_config import ParserConfigIni + + +class SyncPckgMgmt(object): + """ + keep pckg-mgmt.yaml and obs_meta in sync + """ + def __init__(self, **kwargs): + """ + giteeuser: gitee user name + giteeuserpwd: gitee password + """ + self.kwargs = kwargs + self.giteeuser = self.kwargs['gitee_user'] + self.giteeuserpwd = self.kwargs['gitee_pwd'] + self.obs_meta_path = self.kwargs['obs_meta_path'] + self.release_management_path = self.kwargs['release_management_path'] + par = ParserConfigIni() + self.obs_ignored_package = par.get_obs_ignored_package() + + def _get_change_file(self): + """ + get release-managemnet change file + """ + if os.path.exists(self.release_management_path): + os.chdir(self.release_management_path) + cmd = "git diff --name-status HEAD~1 HEAD~0 | grep pckg-mgmt.yaml" + result = os.popen(cmd).read().split('\n') + change_file = [x for x in result if x != ''] + return change_file + else: + log.error("%s not exist!" % self.release_management_path) + sys.exit(1) + + def _get_yaml_file_msg(self, file_path): + """ + get pckg-mgmt.yaml file msg + """ + file_msg = None + if os.path.exists(file_path): + with open(file_path, "r", encoding='utf-8') as f: + file_msg = yaml.load(f, Loader=yaml.FullLoader) + return file_msg + + def _check_pckg_yaml(self, file_msg, branch): + """ + check pckg-mgmt.yaml file + """ + proj = branch.replace("-", ":") + flag = False + for title in file_msg['packages']: + for msg in file_msg['packages'][title]: + if msg['branch_to'] != branch: + flag = True + log.error("%s branch_to is error, please check yaml." % msg['name']) + if not msg['obs_to'].startswith(proj): + flag = True + log.error("%s obs_to is error, please check yaml." % msg['name']) + return flag + + def _parse_yaml_msg(self, file_msg, yaml_type): + """ + parse yaml file msg + """ + tmp = {} + msg = [] + del_msg = [] + prj_pkg = {} + if yaml_type == "old": + for pckg in file_msg['packages']['natural']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) + elif yaml_type == 'multi-new': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['source_dir'], + 'branch_to': pckg['destination_dir'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['destination_dir'] + msg.append(tmp) + elif yaml_type == 'multi-new-delete': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name']} + msg.append(tmp) + else: + for label in file_msg['packages']['everything']: + for pckg in file_msg['packages']['everything'][label]: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + for pckg in file_msg['packages']['epol']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + return msg, del_msg, prj_pkg + + def _write_prj_meta_file(self, file_path, proj): + """ + write project meta file + """ + if "Epol" in proj: + main_proj = proj.replace(':Epol', '') + con_proj = main_proj.replace(':', '_').lower() + epol_repo_aarch64 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_aarch64\"/>" % ( + main_proj, con_proj) + epol_repo_x86 = "\n <path project=\"%s:selfbuild:BaseOS\" repository=\"%s_epol_x86_64\"/>" % ( + main_proj, con_proj) + else: + main_proj = proj + con_proj = main_proj.replace(':', '_').lower() + epol_repo_aarch64 = "" + epol_repo_x86 = "" + # if 'openEuler:Testing' in proj:#Testing + if 'master' in file_path: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="standard_aarch64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <useforbuild> + <disable/> + </useforbuild> + <repository name="standard_aarch64"> + <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_aarch64"/>{3} + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64"> + <path project="{1}:selfbuild:BaseOS" repository="{2}_standard_x86_64"/>{4} + <arch>x86_64</arch> + </repository> +</project> + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + f = open(file_path, "w") + f.write(file_msg) + f.close() + + def _write_selfbuild_meta_file(self, file_path, proj): + """ + write selfbuild project meta file + """ + file_msg = """<project name="{0}:selfbuild:BaseOS"> + <title/> + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="{1}_standard_aarch64"> + <arch>aarch64</arch> + </repository> + <repository name="{1}_standard_x86_64"> + <arch>x86_64</arch> + </repository> + <repository name="{1}_epol_aarch64"> + <arch>aarch64</arch> + </repository> + <repository name="{1}_epol_x86_64"> + <arch>x86_64</arch> + </repository> +</project> +""".format(proj, proj.lower().replace(':', '_')) + f = open(file_path, "w") + f.write(file_msg) + f.close() + + def _write_service_file(self, filepath, package, pkg_branch): + ''' + write service file + ''' + file_msg = """<services> + <service name="tar_scm_kernel_repo"> + <param name="scm">repo</param> + <param name="url">next/{}/{}</param> + </service> +</services>""".format(pkg_branch,package) + try: + with open(os.path.join(filepath,'_service'),'w') as f: + f.write(file_msg) + except Exception as e: + print (e) + + def _add_pkg_service(self, tmp): + """ + add obs_meta packages _service file + """ + success_pkg_name = '' + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if tmp['branch_from'] == "master" and tmp['pkgname'] not in self.obs_ignored_package: + self._write_service_file(pkg_path, tmp['pkgname'], tmp['branch_to']) + if os.path.exists(pkg_service_path): + success_pkg_name = tmp['pkgname'] + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_service_path) and os.path.exists(from_pkg_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + elif not os.path.exists(pkg_service_path) and not os.path.exists(from_pkg_path): + self._write_service_file(pkg_path, tmp['pkgname'], tmp['branch_to']) + if os.path.exists(pkg_service_path): + success_pkg_name = tmp['pkgname'] + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return success_pkg_name + + def _move_pkg_service(self, tmp): + """ + move obs_meta packages _service file + """ + success_pkg_name = '' + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + if not os.path.exists(mv_to_path): + os.makedirs(mv_to_path) + if not os.path.exists(pkg_path): + cmd = "mv %s %s" % (from_pkg_path, mv_to_path) + if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] + log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return success_pkg_name + + def _add_master_pkg_service(self,package): + """ + create and add master service to repo obs_meta + """ + filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) + if not os.path.exists(filepath): + os.makedirs(filepath) + file_msg = """<services> + <service name="tar_scm"> + <param name="scm">git</param> + <param name="url">git@gitee.com:src-openeuler/{}.git</param> + <param name="exclude">*</param> + <param name="extract">*</param> + <param name="revision">master</param> + </service> +</services>""".format(package) + try: + with open(os.path.join(filepath,'_service'),'w') as f: + f.write(file_msg) + log.info("add openEuler:Factory {} _service success!" % (package)) + except Exception as e: + print (e) + log.info("add %s _service failed!" % (filepath)) + + def _move_master_pkg_service(self, msg): + """ + copy branch master obs_meta packages _service file + """ + change_pkgs = [] + for tmp in msg: + if tmp['obs_to'] == 'openEuler:Factory': + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if not os.path.exists(pkg_path): + self._add_master_pkg_service(tmp['pkgname']) + change_pkgs.append(tmp['pkgname']) + else: + pkg_name = self._move_pkg_service(tmp) + change_pkgs.append(pkg_name) + return change_pkgs + + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): + pkg_names = [] + for tmp in msg: + if 'multi_version' not in branch_infos: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'master' not in branch_infos: + if tmp['branch_from'] == tmp['branch_to'] and tmp['obs_from'] != tmp['obs_to']: + pkg_name = self._move_pkg_service(tmp) + else: + pkg_name = self._add_pkg_service(tmp) + if pkg_name: + pkg_names.append(pkg_name) + return pkg_names + + + def _del_pkg(self, tmp): + """ + delete obs_meta packages + """ + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] + + def _del_pkg_new(self, pkgs, branch): + """ + find in branch dirs and delete pkg + """ + need_del_path = [] + correct_del_pkgs = [] + release_changes = {} + branch_dir = os.path.join(self.obs_meta_path,branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path and 'bringInRely' not in c_path and 'RISC-V' not in c_path: + correct_del_pkgs.append(name) + need_del_path.append(c_path) + for pkg_path in need_del_path: + if os.path.exists(pkg_path): + pkg_path_infos = pkg_path.split('/') + shutil.rmtree(pkg_path) + log.info("delete %s %s succeed!" % (branch,pkg_path_infos[-1])) + release_branch = pkg_path_infos[-3] + if release_branch == 'master': + obs_project = pkg_path_infos[-2].replace(":","-") + release_path = os.path.join(self.release_management_path,release_branch,obs_project) + if release_changes.get(release_path): + release_changes[release_path].append(pkg_path_infos[-1]) + else: + release_changes[release_path] = [pkg_path_infos[-1]] + else: + standard_dirs = ['epol','everything-exclude-baseos','baseos'] + for c_dir in standard_dirs: + release_path = os.path.join(self.release_management_path,release_branch,c_dir) + if release_changes.get(release_path): + release_changes[release_path].append(pkg_path_infos[-1]) + else: + release_changes[release_path] = [pkg_path_infos[-1]] + return correct_del_pkgs,release_changes + + def _del_pckg_from_yaml(self, release_changes): + if release_changes: + for pkgs_path,pkgs in release_changes.items(): + result = {} + change_flag = False + complete_path = os.path.join(pkgs_path,'pckg-mgmt.yaml') + old_yaml = self._get_yaml_file_msg(complete_path) + pkgs_list = old_yaml['packages'] + for i in range(len(pkgs_list)-1, -1, -1): + if pkgs_list[i]['name'] in pkgs: + pkgs_list.pop(i) + change_flag = True + if change_flag: + result['packages'] = pkgs_list + with open(complete_path, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + + + def _verify_meta_file(self, prj_pkg): + """ + verify obs_meta with pckg-mgmt.yaml + """ + for proj, pkg in prj_pkg.items(): + if proj != "branch": + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + if len(list(prj_pkg.keys())) >= 3: + need_del_pkg = [] + else: + need_del_pkg = set(meta_pkglist) - set(pkg) + log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) + if need_del_pkg: + for del_pkg in need_del_pkg: + tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} + self._del_pkg(tmp) + return list(need_del_pkg) + + + def _push_code(self,repo): + """ + push code to gitee repo + """ + if os.path.exists(repo): + os.chdir(repo) + cmd = "git status -s" + if os.popen(cmd).read(): + cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" + if os.system(cmd) == 0: + cmd = "git push -f" + for i in range(5): + if os.system(cmd) == 0: + log.info("push code to gitee repo succeed!") + return 0 + else: + log.error("push code failed, try again...") + raise SystemExit("push code to gitee repo Failed!") + else: + log.info("No change, nothing to commit!") + return "nothing to push" + else: + log.error("%s not exist!" % repo) + return -1 + + def _write_release_yaml(self, change_pkgs, branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs and len(change_pkgs) < 50: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + else: + pull_request = commitid + log.info("ignore write release change yaml file success") + + + def sync_yaml_meta(self): + """ + integration of functions + """ + change_file = self._get_change_file() + master_change_file = [] + yaml_dict = {} + master_prj_pkg = [] + for line in change_file: + log.info("line:%s" % line) + name = list(line.split())[1] + file_path = os.path.join(self.release_management_path, name) + yaml_dict = self._get_yaml_file_msg(file_path) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = branch_infos[1] + if branch != 'delete': + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + move_change_pkgs = self._move_master_pkg_service(msg) + self._write_release_yaml(move_change_pkgs,branch_infos[0]) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") + pkgs = [tmp['pkgname'] for tmp in msg] + del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs,'{}'.format(branch_infos[0])) + self._write_release_yaml(del_change_pkgs,branch_infos[0]) + self._del_pckg_from_yaml(yaml_changes) + elif 'multi_version' in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = pkg_names + del_change_pkgs + complete_path = os.path.join(branch_infos[0], branch_infos[1]) + self._write_release_yaml(change_pkgs,complete_path) + else: + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + if 'delete' not in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + self._write_release_yaml(pkg_names,branch_infos[0]) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") + pkgs = [tmp['pkgname'] for tmp in msg] + del_change_pkgs, yaml_changes = self._del_pkg_new(pkgs, branch_infos[0]) + self._write_release_yaml(del_change_pkgs,branch_infos[0]) + self._del_pckg_from_yaml(yaml_changes) + else: + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + for tmp in del_msg: + self._del_pkg(tmp) + del_change_pkgs = self._verify_meta_file(prj_pkg) + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) + return ret + + +if __name__ == "__main__": + kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + mgmt = SyncPckgMgmt(**kw) mgmt.sync_yaml_meta() \ No newline at end of file -- Gitee From 9588280af4002c0c862fc5aa727f3f3092e2326d Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 20 Jul 2022 10:21:02 +0800 Subject: [PATCH 03/19] fix code --- core/sync_pckg_mgmt.py | 56 +++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index c333c8a..549aace 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -249,9 +249,9 @@ class SyncPckgMgmt(object): <param name="scm">repo</param> <param name="url">next/{}/{}</param> </service> -</services>""".format(pkg_branch,package) +</services>""".format(pkg_branch, package) try: - with open(os.path.join(filepath,'_service'),'w') as f: + with open(os.path.join(filepath, '_service'), 'w') as f: f.write(file_msg) except Exception as e: print (e) @@ -262,12 +262,12 @@ class SyncPckgMgmt(object): """ success_pkg_name = '' if 'Multi-Version' in tmp['branch_from']: - dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + dir_name = '{}/{}'.format(tmp['branch_from'], tmp['obs_from']) from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) else: from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + dir_name = '{}/{}'.format(tmp['branch_to'], tmp['obs_to']) pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) else: pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) @@ -320,12 +320,12 @@ class SyncPckgMgmt(object): cmd = "mv %s %s" % (from_pkg_path, mv_to_path) if os.system(cmd) == 0: success_pkg_name = tmp['pkgname'] - log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'], tmp['obs_from'], tmp['obs_to'])) else: - log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'], tmp['obs_from'], tmp['obs_to'])) return success_pkg_name - def _add_master_pkg_service(self,package): + def _add_master_pkg_service(self, package): """ create and add master service to repo obs_meta """ @@ -397,7 +397,7 @@ class SyncPckgMgmt(object): delete obs_meta packages """ if 'Multi-Version' in tmp['branch_to']: - dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + dir_name = '{}/{}'.format(tmp['branch_to'], tmp['obs_to']) pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) else: pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) @@ -413,7 +413,7 @@ class SyncPckgMgmt(object): need_del_path = [] correct_del_pkgs = [] release_changes = {} - branch_dir = os.path.join(self.obs_meta_path,branch) + branch_dir = os.path.join(self.obs_meta_path, branch) for root, dirs, files in os.walk(branch_dir, True): for name in dirs: c_path = os.path.join(root, name) @@ -424,31 +424,31 @@ class SyncPckgMgmt(object): if os.path.exists(pkg_path): pkg_path_infos = pkg_path.split('/') shutil.rmtree(pkg_path) - log.info("delete %s %s succeed!" % (branch,pkg_path_infos[-1])) + log.info("delete %s %s succeed!" % (branch, pkg_path_infos[-1])) release_branch = pkg_path_infos[-3] if release_branch == 'master': - obs_project = pkg_path_infos[-2].replace(":","-") - release_path = os.path.join(self.release_management_path,release_branch,obs_project) + obs_project = pkg_path_infos[-2].replace(":", "-") + release_path = os.path.join(self.release_management_path, release_branch, obs_project) if release_changes.get(release_path): release_changes[release_path].append(pkg_path_infos[-1]) else: release_changes[release_path] = [pkg_path_infos[-1]] else: - standard_dirs = ['epol','everything-exclude-baseos','baseos'] + standard_dirs = ['epol', 'everything-exclude-baseos', 'baseos'] for c_dir in standard_dirs: - release_path = os.path.join(self.release_management_path,release_branch,c_dir) + release_path = os.path.join(self.release_management_path, release_branch, c_dir) if release_changes.get(release_path): release_changes[release_path].append(pkg_path_infos[-1]) else: release_changes[release_path] = [pkg_path_infos[-1]] - return correct_del_pkgs,release_changes + return correct_del_pkgs, release_changes def _del_pckg_from_yaml(self, release_changes): if release_changes: - for pkgs_path,pkgs in release_changes.items(): + for pkgs_path, pkgs in release_changes.items(): result = {} change_flag = False - complete_path = os.path.join(pkgs_path,'pckg-mgmt.yaml') + complete_path = os.path.join(pkgs_path, 'pckg-mgmt.yaml') old_yaml = self._get_yaml_file_msg(complete_path) pkgs_list = old_yaml['packages'] for i in range(len(pkgs_list)-1, -1, -1): @@ -468,7 +468,7 @@ class SyncPckgMgmt(object): for proj, pkg in prj_pkg.items(): if proj != "branch": if 'Multi-Version' in prj_pkg['branch']: - dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + dir_name = '{}/{}'.format(prj_pkg['branch'], proj) meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) meta_pkglist = os.listdir(meta_path) else: @@ -487,7 +487,7 @@ class SyncPckgMgmt(object): return list(need_del_pkg) - def _push_code(self,repo): + def _push_code(self, repo): """ push code to gitee repo """ @@ -565,35 +565,35 @@ class SyncPckgMgmt(object): branch = branch_infos[1] if branch != 'delete': msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") - pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) move_change_pkgs = self._move_master_pkg_service(msg) - self._write_release_yaml(move_change_pkgs,branch_infos[0]) + self._write_release_yaml(move_change_pkgs, branch_infos[0]) else: msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") pkgs = [tmp['pkgname'] for tmp in msg] - del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs,'{}'.format(branch_infos[0])) - self._write_release_yaml(del_change_pkgs,branch_infos[0]) + del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs, '{}'.format(branch_infos[0])) + self._write_release_yaml(del_change_pkgs, branch_infos[0]) self._del_pckg_from_yaml(yaml_changes) elif 'multi_version' in branch_infos: msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") - pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) del_change_pkgs = self._verify_meta_file(prj_pkg) change_pkgs = pkg_names + del_change_pkgs complete_path = os.path.join(branch_infos[0], branch_infos[1]) - self._write_release_yaml(change_pkgs,complete_path) + self._write_release_yaml(change_pkgs, complete_path) else: if not yaml_dict: log.info("%s file content is empty!" % name) elif isinstance(yaml_dict['packages'], list): if 'delete' not in branch_infos: msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") - pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) - self._write_release_yaml(pkg_names,branch_infos[0]) + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + self._write_release_yaml(pkg_names, branch_infos[0]) else: msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") pkgs = [tmp['pkgname'] for tmp in msg] del_change_pkgs, yaml_changes = self._del_pkg_new(pkgs, branch_infos[0]) - self._write_release_yaml(del_change_pkgs,branch_infos[0]) + self._write_release_yaml(del_change_pkgs, branch_infos[0]) self._del_pckg_from_yaml(yaml_changes) else: if "everything" in yaml_dict['packages'].keys(): -- Gitee From fb4e0764ea20b7f6ca9ffb59ea61f38920116ac8 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Thu, 21 Jul 2022 15:02:00 +0800 Subject: [PATCH 04/19] sda --- core/check_release_management.py | 73 ++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 77143b0..fda8a49 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -464,7 +464,7 @@ class CheckReleaseManagement(object): error_flag = False for branch,pkgs in add_infos.items(): log.info('branch:{}'.format(branch)) - log.info('pkgs:{}'.format(pkgs)) + # log.info('pkgs:{}'.format(pkgs)) if branch == 'openEuler-Factory': branch = branch.replace("-",":") for pkg in pkgs: @@ -479,12 +479,15 @@ class CheckReleaseManagement(object): for pkg in pkgs: if 'Multi-Version' in branch: branch = branch.replace("_",":") - from_result = self._find_master_meta_path(pkg,ctype='multi-from') + from_result = self._find_master_meta_path(pkg, ctype='multi-from') move_result = False else: branch = branch.replace("-",":") from_result = self._find_master_meta_path(pkg) - move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) + if not move_infos: + move_result = False + else: + move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) if pkg['obs_to'] != branch: error_flag = True log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) @@ -603,7 +606,6 @@ class CheckReleaseManagement(object): for pkg in pkgs: to_result = self._find_master_meta_path(pkg, ctype='to') else: - # branch = branch.replace("-",":") for pkg in pkgs: if 'Multi-Version' in branch: branch = branch.replace("_",":") @@ -704,7 +706,6 @@ class CheckReleaseManagement(object): if error_from: for branch, pkgs in error_from.items(): error_names = [pkg['name'] for pkg in pkgs] - print (error_names) for pkg in pkgs: master_branch_name = pkg['obs_from'].replace(":",'-') if branch in branch_result['branch']['master'] and all_master_add.get(master_branch_name,''): @@ -824,27 +825,36 @@ class CheckReleaseManagement(object): same_pkg_flag = False log.info("rpms exists and repeat check") for branch,new_msgs in new_msg.items(): - old_pkg[branch] = [] - new_pkg[branch] = [] - same_pkg[branch] = [] - change_pkgs = [] - old_msgs = old_msg[branch] - for old in old_msgs: - old_pkg[branch].append(old['name']) - for new in new_msgs: - if new['name'] in new_pkg[branch]: - same_pkg_flag = True - same_pkg[branch].append(new['name']) - new_pkg[branch].append(new['name']) - error_branch_pkgs = list(set(old_pkg[branch]).difference(set(new_pkg[branch]))) - if error_branch_pkgs: - error_pkg[branch] = error_branch_pkgs - error_pkg_flag = True - add_pkgs = list(set(new_pkg[branch]).difference(set(old_pkg[branch]))) - for new in new_msgs: - if new['name'] in add_pkgs: - change_pkgs.append(new) - change_infos[branch] = change_pkgs + if old_msg.get(branch, []): + old_pkg[branch] = [] + new_pkg[branch] = [] + same_pkg[branch] = [] + change_pkgs = [] + old_msgs = old_msg[branch] + for old in old_msgs: + old_pkg[branch].append(old['name']) + for new in new_msgs: + if new['name'] in new_pkg[branch]: + same_pkg_flag = True + same_pkg[branch].append(new['name']) + new_pkg[branch].append(new['name']) + error_branch_pkgs = list(set(old_pkg[branch]).difference(set(new_pkg[branch]))) + if error_branch_pkgs: + error_pkg[branch] = error_branch_pkgs + error_pkg_flag = True + add_pkgs = list(set(new_pkg[branch]).difference(set(old_pkg[branch]))) + for new in new_msgs: + if new['name'] in add_pkgs: + change_pkgs.append(new) + change_infos[branch] = change_pkgs + else: + new_pkg[branch] = [] + for new in new_msgs: + if new['name'] in new_pkg[branch]: + same_pkg_flag = True + same_pkg[branch].append(new['name']) + new_pkg[branch].append(new['name']) + change_infos[branch] = new_msgs for branch,pkgs in change_infos.items(): if pkgs: log.info("change in:{}".format(branch)) @@ -948,8 +958,9 @@ class CheckReleaseManagement(object): add_flag = self._check_master_add_rules(add_infos, move_infos) move_flag = self._check_master_move_rules(move_infos) del_flag = self._check_master_del_rules(del_old_master_yaml_msg, del_master_change_yaml_msg) - date_flag = self._check_master_date_rules(add_infos) - if add_flag or move_flag or date_flag or del_flag: + # date_flag = self._check_master_date_rules(add_infos) + # if add_flag or move_flag or date_flag or del_flag: + if add_flag or move_flag or del_flag: raise SystemExit("Please check your commit") if new_version_change_file: log.info(new_version_change_file) @@ -980,8 +991,8 @@ if __name__ == "__main__": kw = {"branch":"master", "gitee_user":"", "gitee_pwd":"", - "pr_id":"", - "obs_meta_path":"***", - "release_management_path":"***"} + "pr_id":"147", + "obs_meta_path":"/home/0712/obs_meta", + "release_management_path":"/home/0712/release-management"} check = CheckReleaseManagement(**kw) check.check_pckg_yaml() \ No newline at end of file -- Gitee From e085e4609e4ecbddf44497ba0eb149934609596c Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Thu, 21 Jul 2022 17:42:02 +0800 Subject: [PATCH 05/19] fix code --- core/check_release_management.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index fda8a49..008c0fd 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -958,9 +958,8 @@ class CheckReleaseManagement(object): add_flag = self._check_master_add_rules(add_infos, move_infos) move_flag = self._check_master_move_rules(move_infos) del_flag = self._check_master_del_rules(del_old_master_yaml_msg, del_master_change_yaml_msg) - # date_flag = self._check_master_date_rules(add_infos) - # if add_flag or move_flag or date_flag or del_flag: - if add_flag or move_flag or del_flag: + date_flag = self._check_master_date_rules(add_infos) + if add_flag or move_flag or date_flag or del_flag: raise SystemExit("Please check your commit") if new_version_change_file: log.info(new_version_change_file) -- Gitee From 9b73ba24d2fa0127dc5f87aed1bde75dabd61f86 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 22 Jul 2022 15:12:55 +0800 Subject: [PATCH 06/19] bug fix --- core/check_release_management.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 008c0fd..723fbfe 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -726,7 +726,9 @@ class CheckReleaseManagement(object): break if error_names: error_flag =True - log.error("branch:{},The {} not exist in obs_meta dir and not in parent branch obs_meta dir too".format(branch,error_names)) + for pkg in pkgs: + if pkg['name'] in error_names: + log.error("branch:{}:The {} not exist in obs_meta from dir {}".format(branch, pkg['name'], pkg['source_dir'], pkg['obs_from'])) return error_flag -- Gitee From efc9aaef8f5be453b57a3779825cd210d453fcc6 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 22 Jul 2022 15:37:30 +0800 Subject: [PATCH 07/19] log fix --- core/check_release_management.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 723fbfe..ee5c4e4 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -463,7 +463,7 @@ class CheckReleaseManagement(object): error_infos = {} error_flag = False for branch,pkgs in add_infos.items(): - log.info('branch:{}'.format(branch)) + log.info('check branch:{} add pkgs obs_from check running...'.format(branch)) # log.info('pkgs:{}'.format(pkgs)) if branch == 'openEuler-Factory': branch = branch.replace("-",":") @@ -528,6 +528,7 @@ class CheckReleaseManagement(object): check master delete pckg-mgmt.yaml pkgs is exist in obs_meta ''' error_flag = False + log.info("master delete pkgs check") for branch,old_pkgs in old_msg.items(): new_pkgs = new_msg[branch] old_pkgs_names = [info['name'] for info in old_pkgs] @@ -599,7 +600,7 @@ class CheckReleaseManagement(object): error_infos = {} error_flag = False for branch,pkgs in delete_infos.items(): - log.info('branch:{}'.format(branch)) + log.info('check branch:{} internal move pkgs running...'.format(branch)) log.info('pkgs:{}'.format(pkgs)) if branch == 'openEuler-Factory': branch = branch.replace("-",":") @@ -631,10 +632,11 @@ class CheckReleaseManagement(object): error_flag = False date = datetime.date.today() today = date.day + log.info("master pkgs date check") for branch,pkgs in infos.items(): if branch != 'delete': + log.info("check branch:{} pkgs date check running...".format(branch)) for pkg in pkgs: - log.info("master_branch:{},{} date check".format(branch,pkg['name'])) yaml_date = int(pkg['date'].split('-')[2]) if today != yaml_date: error_flag = True -- Gitee From 6201f344c7eb90c871c1b4b2e0b41c0fffea097b Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 22 Jul 2022 16:17:12 +0800 Subject: [PATCH 08/19] fix code --- core/check_release_management.py | 74 ++++++++++++++++++-------------- core/sync_pckg_mgmt.py | 12 ++++-- 2 files changed, 50 insertions(+), 36 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index ee5c4e4..a1bba4f 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -464,19 +464,29 @@ class CheckReleaseManagement(object): error_flag = False for branch,pkgs in add_infos.items(): log.info('check branch:{} add pkgs obs_from check running...'.format(branch)) - # log.info('pkgs:{}'.format(pkgs)) - if branch == 'openEuler-Factory': - branch = branch.replace("-",":") - for pkg in pkgs: - if pkg['obs_to'] != branch: - error_flag = True - log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) - if error_infos.get(branch,[]): - error_infos[branch].append(pkg) - else: - error_infos[branch] = [pkg] - else: - for pkg in pkgs: + # if branch == 'openEuler-Factory': + # branch = branch.replace("-",":") + # for pkg in pkgs: + # if pkg['obs_to'] != branch: + # error_flag = True + # move_result = False + # log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) + # if error_infos.get(branch,[]): + # error_infos[branch].append(pkg) + # else: + # error_infos[branch] = [pkg] + # if pkg['obs_from']: + # branch = branch.replace("-",":") + # from_result = self._find_master_meta_path(pkg) + # move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) + # if from_result or move_result: + # error_flag = True + # if error_infos.get(branch,[]): + # error_infos[branch].append(pkg) + # else: + # error_infos[branch] = [pkg] + for pkg in pkgs: + if pkg['obs_from']: if 'Multi-Version' in branch: branch = branch.replace("_",":") from_result = self._find_master_meta_path(pkg, ctype='multi-from') @@ -488,19 +498,19 @@ class CheckReleaseManagement(object): move_result = False else: move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) - if pkg['obs_to'] != branch: - error_flag = True - log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) - if error_infos.get(branch,[]): - error_infos[branch].append(pkg) - else: - error_infos[branch] = [pkg] if from_result or move_result: error_flag = True if error_infos.get(branch,[]): error_infos[branch].append(pkg) else: error_infos[branch] = [pkg] + if pkg['obs_to'] != branch: + error_flag = True + log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] if error_infos: log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag @@ -602,18 +612,18 @@ class CheckReleaseManagement(object): for branch,pkgs in delete_infos.items(): log.info('check branch:{} internal move pkgs running...'.format(branch)) log.info('pkgs:{}'.format(pkgs)) - if branch == 'openEuler-Factory': - branch = branch.replace("-",":") - for pkg in pkgs: + # if branch == 'openEuler-Factory': + # branch = branch.replace("-",":") + # for pkg in pkgs: + # to_result = self._find_master_meta_path(pkg, ctype='to') + # else: + for pkg in pkgs: + if 'Multi-Version' in branch: + branch = branch.replace("_",":") + to_result = self._find_master_meta_path(pkg, ctype='multi-to') + else: + branch = branch.replace("-",":") to_result = self._find_master_meta_path(pkg, ctype='to') - else: - for pkg in pkgs: - if 'Multi-Version' in branch: - branch = branch.replace("_",":") - to_result = self._find_master_meta_path(pkg, ctype='multi-to') - else: - branch = branch.replace("-",":") - to_result = self._find_master_meta_path(pkg, ctype='to') if to_result: error_flag = True if error_infos.get(branch,[]): @@ -730,7 +740,7 @@ class CheckReleaseManagement(object): error_flag =True for pkg in pkgs: if pkg['name'] in error_names: - log.error("branch:{}:The {} not exist in obs_meta from dir {}".format(branch, pkg['name'], pkg['source_dir'], pkg['obs_from'])) + log.error("branch:{}:The {} not exist in obs_meta from dir {}/{}".format(branch, pkg['name'], pkg['source_dir'], pkg['obs_from'])) return error_flag diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 549aace..2ea2ef9 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -356,10 +356,14 @@ class SyncPckgMgmt(object): change_pkgs = [] for tmp in msg: if tmp['obs_to'] == 'openEuler:Factory': - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - if not os.path.exists(pkg_path): - self._add_master_pkg_service(tmp['pkgname']) - change_pkgs.append(tmp['pkgname']) + if tmp['obs_from']: + pkg_name = self._move_pkg_service(tmp) + change_pkgs.append(pkg_name) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if not os.path.exists(pkg_path): + self._add_master_pkg_service(tmp['pkgname']) + change_pkgs.append(tmp['pkgname']) else: pkg_name = self._move_pkg_service(tmp) change_pkgs.append(pkg_name) -- Gitee From 359731c185864ea8d84ba105343a1ee74536bc44 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 22 Jul 2022 16:25:08 +0800 Subject: [PATCH 09/19] 0722 --- core/check_release_management.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index a1bba4f..c518dc3 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -486,13 +486,12 @@ class CheckReleaseManagement(object): # else: # error_infos[branch] = [pkg] for pkg in pkgs: + branch = branch.replace("-",":") if pkg['obs_from']: if 'Multi-Version' in branch: - branch = branch.replace("_",":") from_result = self._find_master_meta_path(pkg, ctype='multi-from') move_result = False else: - branch = branch.replace("-",":") from_result = self._find_master_meta_path(pkg) if not move_infos: move_result = False -- Gitee From 7a8d44fd43a347df926ac148872e58500b868516 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Fri, 22 Jul 2022 17:12:41 +0800 Subject: [PATCH 10/19] 2209 --- core/check_release_management.py | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index c518dc3..a5abb5a 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -464,27 +464,6 @@ class CheckReleaseManagement(object): error_flag = False for branch,pkgs in add_infos.items(): log.info('check branch:{} add pkgs obs_from check running...'.format(branch)) - # if branch == 'openEuler-Factory': - # branch = branch.replace("-",":") - # for pkg in pkgs: - # if pkg['obs_to'] != branch: - # error_flag = True - # move_result = False - # log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) - # if error_infos.get(branch,[]): - # error_infos[branch].append(pkg) - # else: - # error_infos[branch] = [pkg] - # if pkg['obs_from']: - # branch = branch.replace("-",":") - # from_result = self._find_master_meta_path(pkg) - # move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) - # if from_result or move_result: - # error_flag = True - # if error_infos.get(branch,[]): - # error_infos[branch].append(pkg) - # else: - # error_infos[branch] = [pkg] for pkg in pkgs: branch = branch.replace("-",":") if pkg['obs_from']: @@ -493,10 +472,7 @@ class CheckReleaseManagement(object): move_result = False else: from_result = self._find_master_meta_path(pkg) - if not move_infos: - move_result = False - else: - move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) + move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) if from_result or move_result: error_flag = True if error_infos.get(branch,[]): -- Gitee From 6568d7c8884cafddf879458fd812663f249a879d Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 09:25:31 +0800 Subject: [PATCH 11/19] repeat check --- core/check_release_management.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index a5abb5a..6662e6a 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -635,6 +635,22 @@ class CheckReleaseManagement(object): log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag + def _check_master_repeat(self, new_msgs): + ''' + check master branch pkg duplicate + ''' + log.info("master pkgs repeat check") + pkgs = [new['name'] for new in new_msgs] + seen = set() + duplicated = set() + for pkg in pkgs: + if pkg not in seen: + seen.add(pkg) + else: + duplicated.add(pkg) + if duplicated: + log.error("The following {0} packages are duplicated in the master YAML files".format(duplicated)) + raise SystemExit("ERROR: Please check your PR") def _check_pkg_from_new(self, meta_path, change_info): """ @@ -904,7 +920,7 @@ class CheckReleaseManagement(object): all_pack_msg[branch] = all_branch_pkgs return all_pack_msg,all_del_msg - def _get_complete_yaml_pkgs(self,branch): + def _get_complete_yaml_pkgs(self, branch): all_branch_pkgs = [] if os.path.exists(os.path.join(self.manage_path, branch)): standard_dirs = os.listdir(os.path.join(self.manage_path, branch)) @@ -920,7 +936,11 @@ class CheckReleaseManagement(object): all_branch_pkgs.extend(result['packages']) return all_branch_pkgs - + def _get_master_yaml_msg(self, path_list, manage_path): + all_master_pkgs = [] + if path_list: + all_master_pkgs = self._get_complete_yaml_pkgs('master') + return all_master_pkgs def check_pckg_yaml(self): """ @@ -935,6 +955,7 @@ class CheckReleaseManagement(object): change_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) all_master_yaml_msg = self._get_allkey_msg(master_change_file, self.manage_path) master_change_yaml_msg,del_master_change_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + all_master_msg = self._get_master_yaml_msg(master_change_file, self.manage_path) new_version_change_msg,del_new_version_change_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') self._rollback_get_msg(self.manage_path) old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) @@ -944,6 +965,7 @@ class CheckReleaseManagement(object): if master_change_file: log.info(master_change_file) add_infos,move_infos= self._get_move_and_add(old_master_yaml_msg, master_change_yaml_msg) + self._check_master_repeat(all_master_msg) add_flag = self._check_master_add_rules(add_infos, move_infos) move_flag = self._check_master_move_rules(move_infos) del_flag = self._check_master_del_rules(del_old_master_yaml_msg, del_master_change_yaml_msg) -- Gitee From 138d358250695d71e86da3560619f7c6df1fad2c Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 09:32:02 +0800 Subject: [PATCH 12/19] 22 --- core/check_release_management.py | 39 ++++++++++++++++---------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 6662e6a..6877577 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -469,11 +469,12 @@ class CheckReleaseManagement(object): if pkg['obs_from']: if 'Multi-Version' in branch: from_result = self._find_master_meta_path(pkg, ctype='multi-from') - move_result = False + # move_result = False else: from_result = self._find_master_meta_path(pkg) - move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) - if from_result or move_result: + # move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) + # if from_result or move_result: + if from_result error_flag = True if error_infos.get(branch,[]): error_infos[branch].append(pkg) @@ -490,22 +491,22 @@ class CheckReleaseManagement(object): log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag - def _find_in_delete_pkgs(self, branch, pkg, move_infos): - ''' - check move pkg in from branch yaml is deleted - ''' - flag = False - branchs = list(move_infos.keys()) - pkg_from = pkg['obs_from'].replace(":","-") - if pkg_from not in branchs: - log.error("pkg name:{} need delete in from branch:{} yaml!!!".format(pkg['name'],pkg['obs_from'])) - flag = True - else: - pkgnames = [info['name'] for info in move_infos[pkg_from]] - if pkg['name'] not in pkgnames: - log.error("pkg name:{} need delete in from branch:{} yaml!!!".format(pkg['name'],pkg['obs_from'])) - flag = True - return flag + # def _find_in_delete_pkgs(self, branch, pkg, move_infos): + # ''' + # check move pkg in from branch yaml is deleted + # ''' + # flag = False + # branchs = list(move_infos.keys()) + # pkg_from = pkg['obs_from'].replace(":","-") + # if pkg_from not in branchs: + # log.error("pkg name:{} need delete in from branch:{} yaml!!!".format(pkg['name'],pkg['obs_from'])) + # flag = True + # else: + # pkgnames = [info['name'] for info in move_infos[pkg_from]] + # if pkg['name'] not in pkgnames: + # log.error("pkg name:{} need delete in from branch:{} yaml!!!".format(pkg['name'],pkg['obs_from'])) + # flag = True + # return flag def _check_master_del_rules(self, old_msg, new_msg): -- Gitee From e8b13f6c390fa0f79714d24e5c1f7f29b45aef61 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 09:59:41 +0800 Subject: [PATCH 13/19] fix --- core/check_release_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 6877577..d4bfc43 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -474,7 +474,7 @@ class CheckReleaseManagement(object): from_result = self._find_master_meta_path(pkg) # move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) # if from_result or move_result: - if from_result + if from_result: error_flag = True if error_infos.get(branch,[]): error_infos[branch].append(pkg) -- Gitee From b8469eae6b1e9eb75ce211a7978098b359250033 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 10:32:15 +0800 Subject: [PATCH 14/19] dss --- core/sync_pckg_mgmt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 2ea2ef9..af2e6e5 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -346,8 +346,8 @@ class SyncPckgMgmt(object): f.write(file_msg) log.info("add openEuler:Factory {} _service success!" % (package)) except Exception as e: - print (e) - log.info("add %s _service failed!" % (filepath)) + log.error(e) + log.error("add %s _service failed!" % (filepath)) def _move_master_pkg_service(self, msg): """ -- Gitee From 3c6a535b3c2b808a64bcc3d1ac5136cccab9406c Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 10:42:26 +0800 Subject: [PATCH 15/19] sdas --- core/sync_pckg_mgmt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index af2e6e5..22da6c5 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -344,10 +344,10 @@ class SyncPckgMgmt(object): try: with open(os.path.join(filepath,'_service'),'w') as f: f.write(file_msg) - log.info("add openEuler:Factory {} _service success!" % (package)) + log.info("add openEuler:Factory {} _service success!".format(package)) except Exception as e: log.error(e) - log.error("add %s _service failed!" % (filepath)) + log.error("add openEuler:Factory {} _service failed!".format(package)) def _move_master_pkg_service(self, msg): """ -- Gitee From 5c672da1ad21868e2d6343c222e1dd8eae449827 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 14:17:01 +0800 Subject: [PATCH 16/19] ff --- core/sync_pckg_mgmt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 22da6c5..b0474aa 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -358,7 +358,8 @@ class SyncPckgMgmt(object): if tmp['obs_to'] == 'openEuler:Factory': if tmp['obs_from']: pkg_name = self._move_pkg_service(tmp) - change_pkgs.append(pkg_name) + if pkg_name: + change_pkgs.append(pkg_name) else: pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) if not os.path.exists(pkg_path): @@ -366,7 +367,8 @@ class SyncPckgMgmt(object): change_pkgs.append(tmp['pkgname']) else: pkg_name = self._move_pkg_service(tmp) - change_pkgs.append(pkg_name) + if pkg_name: + change_pkgs.append(pkg_name) return change_pkgs -- Gitee From f1bf5df5c953b23ce4ec35cc8550e5d3f5e0e7be Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 14:46:46 +0800 Subject: [PATCH 17/19] b --- core/check_release_management.py | 34 +++++++++++--------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index d4bfc43..0785b1a 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -469,11 +469,8 @@ class CheckReleaseManagement(object): if pkg['obs_from']: if 'Multi-Version' in branch: from_result = self._find_master_meta_path(pkg, ctype='multi-from') - # move_result = False else: from_result = self._find_master_meta_path(pkg) - # move_result = self._find_in_delete_pkgs(branch, pkg, move_infos) - # if from_result or move_result: if from_result: error_flag = True if error_infos.get(branch,[]): @@ -491,24 +488,6 @@ class CheckReleaseManagement(object): log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag - # def _find_in_delete_pkgs(self, branch, pkg, move_infos): - # ''' - # check move pkg in from branch yaml is deleted - # ''' - # flag = False - # branchs = list(move_infos.keys()) - # pkg_from = pkg['obs_from'].replace(":","-") - # if pkg_from not in branchs: - # log.error("pkg name:{} need delete in from branch:{} yaml!!!".format(pkg['name'],pkg['obs_from'])) - # flag = True - # else: - # pkgnames = [info['name'] for info in move_infos[pkg_from]] - # if pkg['name'] not in pkgnames: - # log.error("pkg name:{} need delete in from branch:{} yaml!!!".format(pkg['name'],pkg['obs_from'])) - # flag = True - # return flag - - def _check_master_del_rules(self, old_msg, new_msg): ''' check master delete pckg-mgmt.yaml pkgs is exist in obs_meta @@ -636,11 +615,13 @@ class CheckReleaseManagement(object): log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag - def _check_master_repeat(self, new_msgs): + def _check_master_repeat(self, old_msgs, new_msgs): ''' check master branch pkg duplicate ''' + error_flag = False log.info("master pkgs repeat check") + old_pkgs = [old['name'] for old in old_msgs] pkgs = [new['name'] for new in new_msgs] seen = set() duplicated = set() @@ -649,8 +630,14 @@ class CheckReleaseManagement(object): seen.add(pkg) else: duplicated.add(pkg) + error_master_pkgs = list(set(old_pkgs).difference(set(pkgs))) + if error_master_pkgs: + error_flag = True + log.error("The following {0} packages should not deleted in the master YAML files".format(duplicated)) if duplicated: + error_flag = True log.error("The following {0} packages are duplicated in the master YAML files".format(duplicated)) + if error_flag: raise SystemExit("ERROR: Please check your PR") def _check_pkg_from_new(self, meta_path, change_info): @@ -961,12 +948,13 @@ class CheckReleaseManagement(object): self._rollback_get_msg(self.manage_path) old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) old_master_yaml_msg,del_old_master_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + old_all_master_msg = self._get_master_yaml_msg(master_change_file, self.manage_path) old_new_version_msg,del_old_new_version_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') add_infos = {} if master_change_file: log.info(master_change_file) add_infos,move_infos= self._get_move_and_add(old_master_yaml_msg, master_change_yaml_msg) - self._check_master_repeat(all_master_msg) + self._check_master_repeat(old_all_master_msg, all_master_msg) add_flag = self._check_master_add_rules(add_infos, move_infos) move_flag = self._check_master_move_rules(move_infos) del_flag = self._check_master_del_rules(del_old_master_yaml_msg, del_master_change_yaml_msg) -- Gitee From 51fea987d52adb02554f7dbdb6d80e3a38fecb8c Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 14:57:06 +0800 Subject: [PATCH 18/19] ff --- core/check_release_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 0785b1a..f5dcf48 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -633,7 +633,7 @@ class CheckReleaseManagement(object): error_master_pkgs = list(set(old_pkgs).difference(set(pkgs))) if error_master_pkgs: error_flag = True - log.error("The following {0} packages should not deleted in the master YAML files".format(duplicated)) + log.error("The following {0} packages should not deleted in the master YAML files".format(error_master_pkgs)) if duplicated: error_flag = True log.error("The following {0} packages are duplicated in the master YAML files".format(duplicated)) -- Gitee From cf572e20f0e2134fe935bd794352b7fd12f54cc4 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 23 Jul 2022 15:36:06 +0800 Subject: [PATCH 19/19] ds --- core/check_release_management.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index f5dcf48..359720f 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -762,7 +762,7 @@ class CheckReleaseManagement(object): continue else: error_flag = True - log.error("pkg:{1} souce_dir or destination_dir valid check error".format(pkg['name'])) + log.error("pkg:{} souce_dir or destination_dir valid check error".format(pkg['name'])) if error_flag: raise SystemExit("ERROR: Please ensure the source_dir and destination_dir adapt rules") -- Gitee