diff --git a/config/config.ini b/config/config.ini index 922ee3f8f5ae6ea1febfb4bc327aec5c39e9da26..b67934da2eb9618fe977cc9946c96ac346d7f9eb 100644 --- a/config/config.ini +++ b/config/config.ini @@ -31,7 +31,7 @@ name = ci_check build [package_info_file] name = package_info.csv [branch_proj] -master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras +master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Erlang openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby openEuler-20.03-LTS = openEuler:20.03:LTS openEuler:20.03:LTS:Epol openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1 openEuler:20.03:LTS:SP1:Epol openEuler:20.03:LTS:SP1:Extras openEuler-20.03-LTS-Next = openEuler:20.03:LTS:Next openEuler:20.03:LTS:Next:Epol openEuler:20.03:LTS:Next:Extras @@ -64,9 +64,25 @@ obs_meta = https://gitee.com/src-openeuler/obs_meta obs_pkg_rpms = https://gitee.com/openeuler_latest_rpms/obs_pkg_rpms [obs_project_repos] # replace obs project's ":" by "-" -openEuler-Mainline = openEuler:selfbuild:BaseOS/mainline_standard_aarch64 openEuler:selfbuild:BaseOS/mainline_standard_x86_64 +openEuler-Mainline = oopenEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 openEuler-Epol = openEuler:selfbuild:BaseOS/epol_aarch64 openEuler:selfbuild:BaseOS/epol_x86_64 openEuler-Factory = openEuler:selfbuild:BaseOS/factory_aarch64 openEuler:selfbuild:BaseOS/factory_x86_64 +openEuler-BaseTools = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-C = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Common_Languages_Dependent_Tools = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Erlang = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Golang = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Java = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-KernelSpace = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Lua = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Meson = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-MultiLanguage = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Nodejs = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Ocaml = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Perl = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Python = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Qt = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Ruby = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 openEuler-20.03-LTS = openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_aarch64 openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_x86_64 openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_x86_64 openEuler-20.03-LTS-SP1-Epol = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_x86_64 @@ -109,4 +125,4 @@ path = /srv/obs/build [obs_ignore_package] name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv [obs_include_project] -name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely +name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Erlang openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby diff --git a/core/check_release_management.py b/core/check_release_management.py index 50f5877331674d0b938d7fd06b1b519a0a00ccd2..359720fe4c01f51fee6727c3767c1ad2730176ce 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -118,19 +118,39 @@ class CheckReleaseManagement(object): get the change file for latest commit """ new_file_path = [] + new_versin_file_path = [] + master_new_file_path = [] + multi_version_file_path = [] for line in change_file: log.info("line:%s" % line) log_list = list(line.split()) temp_log_type = log_list[0] if len(log_list) == 3: if "pckg-mgmt.yaml" in log_list[2]: - new_file_path.append(log_list[2]) + if 'master' in log_list[2]: + master_new_file_path.append(log_list[2]) + elif 'multi_version' in log_list[2]: + master_new_file_path.append(log_list[2]) + else: + branch_infos = log_list[2].split('/') + if len(branch_infos) == 3: + new_versin_file_path.append(log_list[2]) + else: + new_file_path.append(log_list[2]) elif len(log_list) == 2: if temp_log_type != "D" and "pckg-mgmt.yaml" in log_list[1]: - new_file_path.append(log_list[1]) - if new_file_path: - log.info(new_file_path) - return new_file_path + if 'master' in log_list[1]: + master_new_file_path.append(log_list[1]) + elif 'multi_version' in log_list[1]: + master_new_file_path.append(log_list[1]) + else: + branch_infos = log_list[1].split('/') + if len(branch_infos) == 3: + new_versin_file_path.append(log_list[1]) + else: + new_file_path.append(log_list[1]) + if new_file_path or master_new_file_path or new_versin_file_path: + return new_file_path,master_new_file_path,new_versin_file_path else: log.info("There are no file need to check!!!") sys.exit() @@ -246,8 +266,7 @@ class CheckReleaseManagement(object): delete_tag = self._check_delete_tag(msg, yaml_all_msg[change]) if delete_tag: continue - msg_path = os.path.join(meta_path, msg['branch_from'], - msg['obs_from'], msg['name']) + msg_path = os.path.join(meta_path, msg['branch_from'],msg['obs_from'], msg['name']) if not os.path.exists(msg_path): yaml_key = os.path.join(msg['branch_from'], msg['obs_from'], msg['name']) @@ -404,35 +423,575 @@ class CheckReleaseManagement(object): log.error("Check the delete group in the {0}!!!".format(info_dict)) raise SystemExit("ERROR:Please check your PR") + def _get_move_and_add(self,old_msg,new_msg): + ''' + get master project add and delete pkgs by compare old and new + ''' + add_infos = {} + delete_infos = {} + for branch,old_pkgs in old_msg.items(): + if old_pkgs: + new_pkgs = new_msg[branch] + old_pkgs_names = [info['name'] for info in old_pkgs] + new_pkgs_names = [info['name'] for info in new_pkgs] + del_names = list(set(old_pkgs_names).difference(set(new_pkgs_names))) + add_names = list(set(new_pkgs_names).difference(set(old_pkgs_names))) + if del_names: + for old in old_pkgs: + if old['name'] in del_names: + if delete_infos.get(branch,[]): + delete_infos[branch].append(old) + else: + delete_infos[branch] = [old] + if add_names: + for new in new_pkgs: + if new['name'] in add_names: + if add_infos.get(branch,[]): + add_infos[branch].append(new) + else: + add_infos[branch] = [new] + else: + add_infos[branch] = new_msg[branch] + return add_infos,delete_infos + + def _check_master_add_rules(self, add_infos, move_infos): + ''' + check master add or internal move pkgs rules + ''' + if add_infos: + log.info("master add pkgs obs_from check") + error_infos = {} + error_flag = False + for branch,pkgs in add_infos.items(): + log.info('check branch:{} add pkgs obs_from check running...'.format(branch)) + for pkg in pkgs: + branch = branch.replace("-",":") + if pkg['obs_from']: + if 'Multi-Version' in branch: + from_result = self._find_master_meta_path(pkg, ctype='multi-from') + else: + from_result = self._find_master_meta_path(pkg) + if from_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if pkg['obs_to'] != branch: + error_flag = True + log.error("pkg name:{2} Wrong obs_to: in !!!".format(pkg['obs_to'],branch,pkg['name'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + def _check_master_del_rules(self, old_msg, new_msg): + ''' + check master delete pckg-mgmt.yaml pkgs is exist in obs_meta + ''' + error_flag = False + log.info("master delete pkgs check") + for branch,old_pkgs in old_msg.items(): + new_pkgs = new_msg[branch] + old_pkgs_names = [info['name'] for info in old_pkgs] + new_pkgs_names = [info['name'] for info in new_pkgs] + del_names = list(set(old_pkgs_names).difference(set(new_pkgs_names))) + add_names = list(set(new_pkgs_names).difference(set(old_pkgs_names))) + if add_names: + branch_dir = os.path.join(self.meta_path, 'master') + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in add_names and 'bringInRely' not in c_path and 'RISC-V' not in c_path: + add_names.remove(name) + if add_names: + error_flag = True + log.error("master branch pkg name:{} you want delete not exist in obs_meta!!!".format(add_names)) + return error_flag + + + def _find_master_meta_path(self, pkg, ctype='from'): + ''' + find obs_form or obs_to in obs_meta path + ''' + if ctype == 'from': + pkg_from_path = os.path.join(self.meta_path, 'master', pkg['obs_from'], pkg['name']) + if not os.path.exists(pkg_from_path): + yaml_key = os.path.join('master',pkg['obs_from'], pkg['name']) + log.error("The {0} not exist in obs_meta".format(yaml_key)) + return True + return False + elif ctype == 'multi-from': + if 'Multi-Version' in pkg['source_dir']: + dir_name = '{}/{}'.format(pkg['source_dir'], pkg['obs_from']) + pkg_from_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) + yaml_key = os.path.join('multi_version', dir_name, pkg['name']) + else: + pkg_from_path = os.path.join(self.meta_path, pkg['source_dir'], pkg['obs_from'], pkg['name']) + yaml_key = os.path.join(pkg['source_dir'], pkg['obs_from'], pkg['name']) + if not os.path.exists(pkg_from_path): + log.error("The {0} not exist in obs_meta".format(yaml_key)) + return True + return False + elif ctype == 'multi-to': + if 'Multi-Version' in pkg['destination_dir']: + dir_name = '{}/{}'.format(pkg['destination_dir'],pkg['obs_to']) + pkg_to_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) + yaml_key = os.path.join('multi_version', dir_name, pkg['name']) + else: + pkg_to_path = os.path.join(self.meta_path, pkg['destination_dir'], pkg['obs_to'], pkg['name']) + yaml_key = os.path.join(pkg['destination_dir'], pkg['obs_to'], pkg['name']) + if not os.path.exists(pkg_to_path): + log.error("The pkg {0} you want delete not exist in obs_meta".format(yaml_key)) + return True + return False + else: + pkg_to_path = os.path.join(self.meta_path, 'master', pkg['obs_to'], pkg['name']) + yaml_key = os.path.join('master', pkg['obs_to'], pkg['name']) + if not os.path.exists(pkg_to_path): + log.error("The pkg {0} you want move not exist in obs_meta".format(yaml_key)) + return True + return False + + def _check_master_move_rules(self, delete_infos): + ''' + check master branch internal move pkgs rule + ''' + if delete_infos: + log.info("master internal move pkgs check") + error_infos = {} + error_flag = False + for branch,pkgs in delete_infos.items(): + log.info('check branch:{} internal move pkgs running...'.format(branch)) + log.info('pkgs:{}'.format(pkgs)) + # if branch == 'openEuler-Factory': + # branch = branch.replace("-",":") + # for pkg in pkgs: + # to_result = self._find_master_meta_path(pkg, ctype='to') + # else: + for pkg in pkgs: + if 'Multi-Version' in branch: + branch = branch.replace("_",":") + to_result = self._find_master_meta_path(pkg, ctype='multi-to') + else: + branch = branch.replace("-",":") + to_result = self._find_master_meta_path(pkg, ctype='to') + if to_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + def _check_master_date_rules(self, infos): + ''' + check date is today + ''' + error_infos = {} + error_flag = False + date = datetime.date.today() + today = date.day + log.info("master pkgs date check") + for branch,pkgs in infos.items(): + if branch != 'delete': + log.info("check branch:{} pkgs date check running...".format(branch)) + for pkg in pkgs: + yaml_date = int(pkg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error(pkg) + log.error("Wrong Date: !!!".format(pkg['date'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + def _check_master_repeat(self, old_msgs, new_msgs): + ''' + check master branch pkg duplicate + ''' + error_flag = False + log.info("master pkgs repeat check") + old_pkgs = [old['name'] for old in old_msgs] + pkgs = [new['name'] for new in new_msgs] + seen = set() + duplicated = set() + for pkg in pkgs: + if pkg not in seen: + seen.add(pkg) + else: + duplicated.add(pkg) + error_master_pkgs = list(set(old_pkgs).difference(set(pkgs))) + if error_master_pkgs: + error_flag = True + log.error("The following {0} packages should not deleted in the master YAML files".format(error_master_pkgs)) + if duplicated: + error_flag = True + log.error("The following {0} packages are duplicated in the master YAML files".format(duplicated)) + if error_flag: + raise SystemExit("ERROR: Please check your PR") + + def _check_pkg_from_new(self, meta_path, change_info): + """ + check add pkg obs_from exist in obs_meta + """ + correct_from_check = {} + error_from_check = {} + error_flag = False + for branch,change in change_info.items(): + log.info("{} pkg obs_from check".format(branch)) + for msg in change: + # log.info("{} pkg obs_from check".format(msg['name'])) + msg_path = os.path.join(meta_path, msg['source_dir'],msg['obs_from'], msg['name']) + yaml_key = os.path.join(msg['source_dir'],msg['obs_from'], msg['name']) + if not os.path.exists(msg_path): + error_from_check.setdefault(branch,[]).append(msg) + else: + correct_from_check.setdefault(branch,[]).append(msg) + return correct_from_check,error_from_check + + def _check_pkg_delete_new(self, meta_path, change_info): + """ + check delete pkg exist in obs_meta + """ + error_flag = False + for branch,change in change_info.items(): + log.info("{} pkg obs_from check".format(branch)) + if change: + pkgs = [msg['name'] for msg in change] + log.info("The {0} exist in obs_meta dir {1} check".format(pkgs,branch)) + branch_dir = os.path.join(meta_path,branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path: + pkgs.remove(name) + if pkgs: + log.error("The {0} not exist in obs_meta dir {1}".format(pkgs,branch)) + error_flag = True + return error_flag + + + def _check_pkg_parent_from(self, change_info, correct_from, error_from, add_infos): + ''' + re-check error pkg_from and find in parent branch add + ''' + error_flag = False + branch_msg_path = os.path.join(self.manage_path, "valid_release_branches.yaml") + with open(branch_msg_path, 'r', encoding='utf-8') as f: + branch_result = yaml.load(f, Loader=yaml.FullLoader) + all_master_add = {} + if add_infos: + for master_branch, master_pkgs in add_infos.items(): + master_pkgnames = [line['name'] for line in master_pkgs] + all_master_add[master_branch] = master_pkgnames + if error_from: + for branch, pkgs in error_from.items(): + error_names = [pkg['name'] for pkg in pkgs] + for pkg in pkgs: + master_branch_name = pkg['obs_from'].replace(":",'-') + if branch in branch_result['branch']['master'] and all_master_add.get(master_branch_name,''): + temps = all_master_add[master_branch_name] + if pkg['name'] in temps: + error_names.remove(pkg['name']) + else: + parent_branch = '' + for valied_parent,child_branchs in branch_result['branch'].items(): + if branch in child_branchs: + parent_branch = valied_parent + break + if parent_branch: + if correct_from.get(parent_branch,''): + correct_branch_names = [line['name'] for line in correct_from[parent_branch]] + if pkg['name'] in correct_branch_names: + error_names.remove(pkg['name']) + break + if error_names: + error_flag =True + for pkg in pkgs: + if pkg['name'] in error_names: + log.error("branch:{}:The {} not exist in obs_meta from dir {}/{}".format(branch, pkg['name'], pkg['source_dir'], pkg['obs_from'])) + return error_flag + + + def _check_key_in_yaml_new(self, change_info): + """ + check the key and brach from in your yaml compliance with rules + """ + error_flag = False + keylist = ['source_dir', 'obs_from', 'name', 'destination_dir', 'obs_to', 'date'] + for branch,info in change_info.items(): + if info: + log.info("branch:{} yaml key check".format(branch)) + for msg in info: + if len(msg.keys()) == 7 or len(msg.keys()) == 6: + for key in msg.keys(): + if key not in keylist: + error_flag = True + log.error(msg) + log.error("ERROR:<<<<<<{0}:>>>>>> should not in there".format(key)) + else: + error_flag = True + log.error("Please check {0}".format(msg)) + if error_flag: + raise SystemExit("ERROR: Please ensure the following key values in your yaml") + + def _check_valid_release_branch(self, change_info): + """ + check the source_dir and destination_dir in your yaml compliance with rules + """ + error_flag = False + branch_msg_path = os.path.join(self.manage_path, "valid_release_branches.yaml") + with open(branch_msg_path, 'r', encoding='utf-8') as f: + branch_result = yaml.load(f, Loader=yaml.FullLoader) + for branch,msg in change_info.items(): + log.info("{} source_dir and destination_dir valid check".format(branch)) + for pkg in msg: + if pkg['destination_dir'] == branch and \ + pkg['source_dir'] in branch_result['branch'].keys() and \ + pkg['destination_dir'] in branch_result['branch'][pkg['source_dir']]: + continue + else: + error_flag = True + log.error("pkg:{} souce_dir or destination_dir valid check error".format(pkg['name'])) + if error_flag: + raise SystemExit("ERROR: Please ensure the source_dir and destination_dir adapt rules") + + def _check_pkg_date(self, change_info): + ''' + check new version pkgs date + ''' + error_flag = False + date = datetime.date.today() + today = date.day + for branch,msg in change_info.items(): + log.info("{0} date check".format(branch)) + for pkg in msg: + yaml_date = int(pkg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error("Wrong Date: !!!".format(pkg['date'])) + return error_flag + + + def _ensure_delete_infos(self, del_old_msg, del_new_msg): + ''' + check new version delete pkgs + ''' + del_new_pkg = {} + del_old_pkg = {} + ensure_delete_pkg = {} + for branch,del_new_msgs in del_new_msg.items(): + del_change_pkgs = [] + del_new_pkg[branch] = [new['name'] for new in del_new_msgs] + del_old_pkg[branch] = [old['name'] for old in del_old_msg[branch]] + delete_pkgs = list(set(del_new_pkg[branch]).difference(set(del_old_pkg[branch]))) + if delete_pkgs: + for del_new in del_new_msgs: + if del_new['name'] in delete_pkgs: + del_change_pkgs.append(del_new) + ensure_delete_pkg[branch] = del_change_pkgs + for branch,branch_del_pkgs in ensure_delete_pkg.items(): + if branch_del_pkgs: + log.info("branch:{},delete pkgs:{}".format(branch,branch_del_pkgs)) + return ensure_delete_pkg + + def _check_rpms_complete_and_repeat(self, old_msg, new_msg): + ''' + compare with old and new yaml msg, make sure package exist + ''' + old_pkg = {} + new_pkg = {} + same_pkg = {} + error_pkg = {} + change_infos = {} + error_pkg_flag = False + same_pkg_flag = False + log.info("rpms exists and repeat check") + for branch,new_msgs in new_msg.items(): + if old_msg.get(branch, []): + old_pkg[branch] = [] + new_pkg[branch] = [] + same_pkg[branch] = [] + change_pkgs = [] + old_msgs = old_msg[branch] + for old in old_msgs: + old_pkg[branch].append(old['name']) + for new in new_msgs: + if new['name'] in new_pkg[branch]: + same_pkg_flag = True + same_pkg[branch].append(new['name']) + new_pkg[branch].append(new['name']) + error_branch_pkgs = list(set(old_pkg[branch]).difference(set(new_pkg[branch]))) + if error_branch_pkgs: + error_pkg[branch] = error_branch_pkgs + error_pkg_flag = True + add_pkgs = list(set(new_pkg[branch]).difference(set(old_pkg[branch]))) + for new in new_msgs: + if new['name'] in add_pkgs: + change_pkgs.append(new) + change_infos[branch] = change_pkgs + else: + new_pkg[branch] = [] + for new in new_msgs: + if new['name'] in new_pkg[branch]: + same_pkg_flag = True + same_pkg[branch].append(new['name']) + new_pkg[branch].append(new['name']) + change_infos[branch] = new_msgs + for branch,pkgs in change_infos.items(): + if pkgs: + log.info("change in:{}".format(branch)) + for pkg in pkgs: + log.info(pkg) + if error_pkg_flag: + log.error("May be {0} should not be delete".format(error_pkg)) + raise SystemExit("ERROR: Please check your PR") + if same_pkg_flag: + log.error("The following {0} packages are duplicated in the YAML files".format(same_pkg)) + raise SystemExit("ERROR: Please check your PR") + return change_infos + + def _get_new_version_yaml_msg(self, yaml_path_list, manage_path,vtype='master'): + ''' + get new version yaml msg content + ''' + all_pack_msg = {} + all_del_msg = {} + for yaml_path in yaml_path_list: + result = {} + file_path = os.path.join(manage_path, yaml_path) + if vtype == 'master': + branch_infos = yaml_path.split('/') + branch = branch_infos[1] + if os.path.exists(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + yaml_packages = [pkg for pkg in result['packages']] + if 'delete' in branch_infos: + all_del_msg[branch] = yaml_packages + else: + if all_pack_msg.get(branch,''): + full_packags = all_pack_msg[branch] + yaml_packages + all_pack_msg[branch] = full_packags + else: + all_pack_msg[branch] = yaml_packages + else: + if 'delete' in branch_infos: + all_del_msg[branch] = [] + else: + all_pack_msg[branch] = [] + else: + branch_infos = yaml_path.split('/') + branch = branch_infos[0] + if 'delete' in branch_infos: + if os.path.exists(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + yaml_packages = [pkg for pkg in result['packages']] + all_del_msg[branch] = yaml_packages + else: + all_del_msg[branch] = [] + else: + if not all_pack_msg.get(branch,''): + all_branch_pkgs = self._get_complete_yaml_pkgs(branch) + if all_branch_pkgs: + all_pack_msg[branch] = all_branch_pkgs + return all_pack_msg,all_del_msg + + def _get_complete_yaml_pkgs(self, branch): + all_branch_pkgs = [] + if os.path.exists(os.path.join(self.manage_path, branch)): + standard_dirs = os.listdir(os.path.join(self.manage_path, branch)) + for standard_dir in standard_dirs: + file_path = os.path.join(self.manage_path, branch, standard_dir) + if not os.path.isdir(file_path) or standard_dir == 'delete': + standard_dirs.remove(standard_dir) + for c_dir in standard_dirs: + release_path = os.path.join(self.manage_path,branch, c_dir, 'pckg-mgmt.yaml') + if os.path.exists(release_path): + with open(release_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + all_branch_pkgs.extend(result['packages']) + return all_branch_pkgs + + def _get_master_yaml_msg(self, path_list, manage_path): + all_master_pkgs = [] + if path_list: + all_master_pkgs = self._get_complete_yaml_pkgs('master') + return all_master_pkgs + def check_pckg_yaml(self): """ check the obs_from branch_from in pckg-mgmt.yaml """ change = self._get_repo_change_file('openeuler', 'release-management', self.manage_path) - change_file = self._parse_commit_file(change) - self._check_yaml_format(change_file, self.manage_path) + change_file,master_change_file,new_version_change_file = self._parse_commit_file(change) + all_change_files = [*change_file, *master_change_file, *new_version_change_file] + self._check_yaml_format(all_change_files, self.manage_path) all_yaml_msg = self._get_allkey_msg(change_file, self.manage_path) change_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) - old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path, True) - self._check_rpms_integrity(old_yaml_msg, change_yaml_msg, change_file) - change_msg_list = self._get_diff_msg(old_yaml_msg, change_yaml_msg, change_file) - log.info(len(change_msg_list)) - self._ensure_delete_tags(change_msg_list, old_yaml_msg, all_yaml_msg) - self._check_key_in_yaml(change_msg_list, change_file) - error_flag1 = self._check_pkg_from(self.meta_path, change_msg_list, change_file, all_yaml_msg) - error_flag2 = self._check_date_time(change_msg_list, change_file) - error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) - error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) - if error_flag1 or error_flag2 or error_flag3 or error_flag4: - raise SystemExit("Please check your commit") + all_master_yaml_msg = self._get_allkey_msg(master_change_file, self.manage_path) + master_change_yaml_msg,del_master_change_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + all_master_msg = self._get_master_yaml_msg(master_change_file, self.manage_path) + new_version_change_msg,del_new_version_change_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') + self._rollback_get_msg(self.manage_path) + old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) + old_master_yaml_msg,del_old_master_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + old_all_master_msg = self._get_master_yaml_msg(master_change_file, self.manage_path) + old_new_version_msg,del_old_new_version_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') + add_infos = {} + if master_change_file: + log.info(master_change_file) + add_infos,move_infos= self._get_move_and_add(old_master_yaml_msg, master_change_yaml_msg) + self._check_master_repeat(old_all_master_msg, all_master_msg) + add_flag = self._check_master_add_rules(add_infos, move_infos) + move_flag = self._check_master_move_rules(move_infos) + del_flag = self._check_master_del_rules(del_old_master_yaml_msg, del_master_change_yaml_msg) + date_flag = self._check_master_date_rules(add_infos) + if add_flag or move_flag or date_flag or del_flag: + raise SystemExit("Please check your commit") + if new_version_change_file: + log.info(new_version_change_file) + change_infos = self._check_rpms_complete_and_repeat(old_new_version_msg, new_version_change_msg) + change_delete_infos = self._ensure_delete_infos(del_old_new_version_msg, del_new_version_change_msg) + self._check_key_in_yaml_new(change_infos) + self._check_valid_release_branch(change_infos) + date_flag = self._check_pkg_date(change_infos) + correct_from, error_from = self._check_pkg_from_new(self.meta_path, change_infos) + error_flag_add = self._check_pkg_parent_from(change_infos, correct_from, error_from, add_infos) + error_flag_del = self._check_pkg_delete_new(self.meta_path, change_delete_infos) + if error_flag_add or error_flag_del or date_flag: + raise SystemExit("Please check your commit") + if change_file: + log.info(change_file) + self._check_rpms_integrity(old_yaml_msg, change_yaml_msg, change_file) + change_msg_list = self._get_diff_msg(old_yaml_msg, change_yaml_msg, change_file) + self._ensure_delete_tags(change_msg_list, old_yaml_msg, all_yaml_msg) + self._check_key_in_yaml(change_msg_list, change_file) + error_flag1 = self._check_pkg_from(self.meta_path, change_msg_list, change_file, all_yaml_msg) + error_flag2 = self._check_date_time(change_msg_list, change_file) + error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) + error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) + if error_flag1 or error_flag2 or error_flag3 or error_flag4: + raise SystemExit("Please check your commit") if __name__ == "__main__": kw = {"branch":"master", "gitee_user":"", "gitee_pwd":"", - "pr_id":"", - "obs_meta_path":"***", - "release_management_path":"***"} + "pr_id":"147", + "obs_meta_path":"/home/0712/obs_meta", + "release_management_path":"/home/0712/release-management"} check = CheckReleaseManagement(**kw) - check.check_pckg_yaml() + check.check_pckg_yaml() \ No newline at end of file diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index ebf73f23df88c629ccd827bafe307f2163bd210c..b0474aa1c6ba05db37822767733daeea60ef689e 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -18,9 +18,11 @@ Synchronize the obs_meta file according to the pckg-mgmt.yaml file """ import os +import re import sys import yaml import shutil +import datetime now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(os.path.join(now_path, "..")) from common.log_obs import log @@ -100,6 +102,36 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) + elif yaml_type == 'multi-new': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['source_dir'], + 'branch_to': pckg['destination_dir'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['destination_dir'] + msg.append(tmp) + elif yaml_type == 'multi-new-delete': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name']} + msg.append(tmp) else: for label in file_msg['packages']['everything']: for pckg in file_msg['packages']['everything'][label]: @@ -116,16 +148,16 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) return msg, del_msg, prj_pkg def _write_prj_meta_file(self, file_path, proj): @@ -144,7 +176,24 @@ class SyncPckgMgmt(object): con_proj = main_proj.replace(':', '_').lower() epol_repo_aarch64 = "" epol_repo_x86 = "" - file_msg = """ + # if 'openEuler:Testing' in proj:#Testing + if 'master' in file_path: + file_msg = """ + + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="standard_aarch64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> <title/> <description/> <person userid="Admin" role="maintainer"/> @@ -160,7 +209,7 @@ class SyncPckgMgmt(object): <arch>x86_64</arch> </repository> </project> -""".format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) f = open(file_path, "w") f.write(file_msg) f.close() @@ -200,9 +249,9 @@ class SyncPckgMgmt(object): <param name="scm">repo</param> <param name="url">next/{}/{}</param> </service> -</services>""".format(pkg_branch,package) +</services>""".format(pkg_branch, package) try: - with open(os.path.join(filepath,'_service'),'w') as f: + with open(os.path.join(filepath, '_service'), 'w') as f: f.write(file_msg) except Exception as e: print (e) @@ -211,57 +260,212 @@ class SyncPckgMgmt(object): """ add obs_meta packages _service file """ - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + success_pkg_name = '' + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'], tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'], tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) pkg_service_path = os.path.join(pkg_path, "_service") if not os.path.exists(pkg_path): os.makedirs(pkg_path) - if tmp['branch_from'] == "master" and tmp['pkgname'] not in self.obs_ignored_package: - self._write_service_file(pkg_path, tmp['pkgname'], tmp['branch_to']) - if os.path.exists(pkg_service_path): - log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) - else: - if tmp['branch_from'] == "master": - branch = "openEuler" + if tmp['branch_from'] == "master" and tmp['pkgname'] not in self.obs_ignored_package: + self._write_service_file(pkg_path, tmp['pkgname'], tmp['branch_to']) + if os.path.exists(pkg_service_path): + success_pkg_name = tmp['pkgname'] + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: - branch = tmp['branch_from'] - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if tmp['branch_from'] == "master": + branch = "openEuler" + else: + branch = tmp['branch_from'] + if not os.path.exists(pkg_service_path) and os.path.exists(from_pkg_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) if os.system(cmd) == 0: + cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) + if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] + log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + else: + log.error("copy %s service file failed!" % tmp['pkgname']) + elif not os.path.exists(pkg_service_path) and not os.path.exists(from_pkg_path): + self._write_service_file(pkg_path, tmp['pkgname'], tmp['branch_to']) + if os.path.exists(pkg_service_path): + success_pkg_name = tmp['pkgname'] log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return success_pkg_name + + def _move_pkg_service(self, tmp): + """ + move obs_meta packages _service file + """ + success_pkg_name = '' + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + if not os.path.exists(mv_to_path): + os.makedirs(mv_to_path) + if not os.path.exists(pkg_path): + cmd = "mv %s %s" % (from_pkg_path, mv_to_path) + if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] + log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'], tmp['obs_from'], tmp['obs_to'])) + else: + log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'], tmp['obs_from'], tmp['obs_to'])) + return success_pkg_name + + def _add_master_pkg_service(self, package): + """ + create and add master service to repo obs_meta + """ + filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) + if not os.path.exists(filepath): + os.makedirs(filepath) + file_msg = """<services> + <service name="tar_scm"> + <param name="scm">git</param> + <param name="url">git@gitee.com:src-openeuler/{}.git</param> + <param name="exclude">*</param> + <param name="extract">*</param> + <param name="revision">master</param> + </service> +</services>""".format(package) + try: + with open(os.path.join(filepath,'_service'),'w') as f: + f.write(file_msg) + log.info("add openEuler:Factory {} _service success!".format(package)) + except Exception as e: + log.error(e) + log.error("add openEuler:Factory {} _service failed!".format(package)) + + def _move_master_pkg_service(self, msg): + """ + copy branch master obs_meta packages _service file + """ + change_pkgs = [] + for tmp in msg: + if tmp['obs_to'] == 'openEuler:Factory': + if tmp['obs_from']: + pkg_name = self._move_pkg_service(tmp) + if pkg_name: + change_pkgs.append(pkg_name) else: - log.error("copy %s service file failed!" % tmp['pkgname']) - - def _add_prj_meta_pkgs_service(self, msg): + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if not os.path.exists(pkg_path): + self._add_master_pkg_service(tmp['pkgname']) + change_pkgs.append(tmp['pkgname']) + else: + pkg_name = self._move_pkg_service(tmp) + if pkg_name: + change_pkgs.append(pkg_name) + return change_pkgs + + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): + pkg_names = [] for tmp in msg: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to']: - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - self._add_pkg_service(tmp) + if 'multi_version' not in branch_infos: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'master' not in branch_infos: + if tmp['branch_from'] == tmp['branch_to'] and tmp['obs_from'] != tmp['obs_to']: + pkg_name = self._move_pkg_service(tmp) + else: + pkg_name = self._add_pkg_service(tmp) + if pkg_name: + pkg_names.append(pkg_name) + return pkg_names + def _del_pkg(self, tmp): """ delete obs_meta packages """ - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'], tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) if os.path.exists(pkg_path): shutil.rmtree(pkg_path) log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] + + def _del_pkg_new(self, pkgs, branch): + """ + find in branch dirs and delete pkg + """ + need_del_path = [] + correct_del_pkgs = [] + release_changes = {} + branch_dir = os.path.join(self.obs_meta_path, branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path and 'bringInRely' not in c_path and 'RISC-V' not in c_path: + correct_del_pkgs.append(name) + need_del_path.append(c_path) + for pkg_path in need_del_path: + if os.path.exists(pkg_path): + pkg_path_infos = pkg_path.split('/') + shutil.rmtree(pkg_path) + log.info("delete %s %s succeed!" % (branch, pkg_path_infos[-1])) + release_branch = pkg_path_infos[-3] + if release_branch == 'master': + obs_project = pkg_path_infos[-2].replace(":", "-") + release_path = os.path.join(self.release_management_path, release_branch, obs_project) + if release_changes.get(release_path): + release_changes[release_path].append(pkg_path_infos[-1]) + else: + release_changes[release_path] = [pkg_path_infos[-1]] + else: + standard_dirs = ['epol', 'everything-exclude-baseos', 'baseos'] + for c_dir in standard_dirs: + release_path = os.path.join(self.release_management_path, release_branch, c_dir) + if release_changes.get(release_path): + release_changes[release_path].append(pkg_path_infos[-1]) + else: + release_changes[release_path] = [pkg_path_infos[-1]] + return correct_del_pkgs, release_changes + + def _del_pckg_from_yaml(self, release_changes): + if release_changes: + for pkgs_path, pkgs in release_changes.items(): + result = {} + change_flag = False + complete_path = os.path.join(pkgs_path, 'pckg-mgmt.yaml') + old_yaml = self._get_yaml_file_msg(complete_path) + pkgs_list = old_yaml['packages'] + for i in range(len(pkgs_list)-1, -1, -1): + if pkgs_list[i]['name'] in pkgs: + pkgs_list.pop(i) + change_flag = True + if change_flag: + result['packages'] = pkgs_list + with open(complete_path, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + def _verify_meta_file(self, prj_pkg): """ @@ -269,19 +473,32 @@ class SyncPckgMgmt(object): """ for proj, pkg in prj_pkg.items(): if proj != "branch": - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) - need_del_pkg = set(meta_pkglist) - set(pkg) + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'], proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + if len(list(prj_pkg.keys())) >= 3: + need_del_pkg = [] + else: + need_del_pkg = set(meta_pkglist) - set(pkg) log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) - for del_pkg in need_del_pkg: - tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} - self._del_pkg(tmp) + if need_del_pkg: + for del_pkg in need_del_pkg: + tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} + self._del_pkg(tmp) + return list(need_del_pkg) - def _push_code(self): + + def _push_code(self, repo): """ push code to gitee repo """ - if os.path.exists(self.obs_meta_path): - os.chdir(self.obs_meta_path) + if os.path.exists(repo): + os.chdir(repo) cmd = "git status -s" if os.popen(cmd).read(): cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" @@ -298,38 +515,108 @@ class SyncPckgMgmt(object): log.info("No change, nothing to commit!") return "nothing to push" else: - log.error("%s not exist!" % self.obs_meta_path) + log.error("%s not exist!" % repo) return -1 + def _write_release_yaml(self, change_pkgs, branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs and len(change_pkgs) < 50: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + else: + pull_request = commitid + log.info("ignore write release change yaml file success") + + def sync_yaml_meta(self): """ integration of functions """ change_file = self._get_change_file() + master_change_file = [] yaml_dict = {} + master_prj_pkg = [] for line in change_file: log.info("line:%s" % line) name = list(line.split())[1] - branch = name.split('/')[0] file_path = os.path.join(self.release_management_path, name) yaml_dict = self._get_yaml_file_msg(file_path) - if not yaml_dict: - log.info("%s file content is empty!" % name) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = branch_infos[1] + if branch != 'delete': + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + move_change_pkgs = self._move_master_pkg_service(msg) + self._write_release_yaml(move_change_pkgs, branch_infos[0]) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") + pkgs = [tmp['pkgname'] for tmp in msg] + del_change_pkgs,yaml_changes = self._del_pkg_new(pkgs, '{}'.format(branch_infos[0])) + self._write_release_yaml(del_change_pkgs, branch_infos[0]) + self._del_pckg_from_yaml(yaml_changes) + elif 'multi_version' in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = pkg_names + del_change_pkgs + complete_path = os.path.join(branch_infos[0], branch_infos[1]) + self._write_release_yaml(change_pkgs, complete_path) else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg) + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + if 'delete' not in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + self._write_release_yaml(pkg_names, branch_infos[0]) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") + pkgs = [tmp['pkgname'] for tmp in msg] + del_change_pkgs, yaml_changes = self._del_pkg_new(pkgs, branch_infos[0]) + self._write_release_yaml(del_change_pkgs, branch_infos[0]) + self._del_pckg_from_yaml(yaml_changes) else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg) - for tmp in del_msg: - self._del_pkg(tmp) - self._verify_meta_file(prj_pkg) - ret = self._push_code() + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + for tmp in del_msg: + self._del_pkg(tmp) + del_change_pkgs = self._verify_meta_file(prj_pkg) + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) return ret if __name__ == "__main__": kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} mgmt = SyncPckgMgmt(**kw) - mgmt.sync_yaml_meta() + mgmt.sync_yaml_meta() \ No newline at end of file