diff --git a/config/config.ini b/config/config.ini index 5ae97f32b208ac6ccf8f7757f6e3e698f982dfd5..09846d7be2f0f900249697374e94563347dd42b8 100644 --- a/config/config.ini +++ b/config/config.ini @@ -30,7 +30,7 @@ name = ci_check build [package_info_file] name = package_info.csv [branch_proj] -master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras +master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Epol openEuler:Erlang openEuler:Factory openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Mainline openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby openEuler-20.03-LTS = openEuler:20.03:LTS openEuler:20.03:LTS:Epol openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1 openEuler:20.03:LTS:SP1:Epol openEuler:20.03:LTS:SP1:Extras openEuler-20.03-LTS-Next = openEuler:20.03:LTS:Next openEuler:20.03:LTS:Next:Epol openEuler:20.03:LTS:Next:Extras @@ -65,6 +65,22 @@ obs_pkg_rpms = https://gitee.com/unsunghero/obs_pkg_rpms openEuler-Mainline = openEuler:selfbuild:BaseOS/mainline_standard_aarch64 openEuler:selfbuild:BaseOS/mainline_standard_x86_64 openEuler-Epol = openEuler:selfbuild:BaseOS/epol_aarch64 openEuler:selfbuild:BaseOS/epol_x86_64 openEuler-Factory = openEuler:selfbuild:BaseOS/factory_aarch64 openEuler:selfbuild:BaseOS/factory_x86_64 +openEuler-BaseTools = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-C = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Common_Languages_Dependent_Tools = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Erlang = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Golang = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Java = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-KernelSpace = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Lua = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Meson = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-MultiLanguage = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Nodejs = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Ocaml = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Perl = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Python = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Qt = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Ruby = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 openEuler-20.03-LTS = openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_aarch64 openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_x86_64 openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_x86_64 openEuler-20.03-LTS-SP1-Epol = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_x86_64 @@ -105,4 +121,4 @@ path = /srv/obs/build [obs_ignore_package] name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv [obs_include_project] -name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely \ No newline at end of file +name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Epol openEuler:Erlang openEuler:Factory openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Mainline openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby \ No newline at end of file diff --git a/core/check_release_management.py b/core/check_release_management.py index 50f5877331674d0b938d7fd06b1b519a0a00ccd2..ab96c842b44b53bac0875cde70c36a20dddf92b5 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -118,19 +118,39 @@ class CheckReleaseManagement(object): get the change file for latest commit """ new_file_path = [] + new_versin_file_path = [] + master_new_file_path = [] + multi_version_file_path = [] for line in change_file: log.info("line:%s" % line) log_list = list(line.split()) temp_log_type = log_list[0] if len(log_list) == 3: if "pckg-mgmt.yaml" in log_list[2]: - new_file_path.append(log_list[2]) + if 'master' in log_list[2]: + master_new_file_path.append(log_list[2]) + elif 'multi_version' in log_list[2]: + master_new_file_path.append(log_list[2]) + else: + branch_infos = log_list[2].split('/') + if len(branch_infos) == 3: + new_versin_file_path.append(log_list[2]) + else: + new_file_path.append(log_list[2]) elif len(log_list) == 2: if temp_log_type != "D" and "pckg-mgmt.yaml" in log_list[1]: - new_file_path.append(log_list[1]) - if new_file_path: - log.info(new_file_path) - return new_file_path + if 'master' in log_list[1]: + master_new_file_path.append(log_list[1]) + elif 'multi_version' in log_list[1]: + master_new_file_path.append(log_list[1]) + else: + branch_infos = log_list[1].split('/') + if len(branch_infos) == 3: + new_versin_file_path.append(log_list[1]) + else: + new_file_path.append(log_list[1]) + if new_file_path or master_new_file_path or new_versin_file_path: + return new_file_path,master_new_file_path,new_versin_file_path else: log.info("There are no file need to check!!!") sys.exit() @@ -246,8 +266,7 @@ class CheckReleaseManagement(object): delete_tag = self._check_delete_tag(msg, yaml_all_msg[change]) if delete_tag: continue - msg_path = os.path.join(meta_path, msg['branch_from'], - msg['obs_from'], msg['name']) + msg_path = os.path.join(meta_path, msg['branch_from'],msg['obs_from'], msg['name']) if not os.path.exists(msg_path): yaml_key = os.path.join(msg['branch_from'], msg['obs_from'], msg['name']) @@ -404,35 +423,391 @@ class CheckReleaseManagement(object): log.error("Check the delete group in the {0}!!!".format(info_dict)) raise SystemExit("ERROR:Please check your PR") + def _get_delete_and_add(self,old_msg,new_msg): + ''' + get master project add and delete pkgs by compare old and new + ''' + add_infos = {} + delete_infos = {} + for branch,old_pkgs in old_msg.items(): + new_pkgs = new_msg[branch] + old_pkgs_names = [info['name'] for info in old_pkgs] + new_pkgs_names = [info['name'] for info in new_pkgs] + del_names = list(set(old_pkgs_names).difference(set(new_pkgs_names))) + add_names = list(set(new_pkgs_names).difference(set(old_pkgs_names))) + if del_names: + for old in old_pkgs: + if old['name'] in del_names: + if delete_infos.get(branch,[]): + delete_infos[branch].append(old) + else: + delete_infos[branch] = [old] + if add_names: + for new in new_pkgs: + if new['name'] in add_names: + if add_infos.get(branch,[]): + add_infos[branch].append(new) + else: + add_infos[branch] = [new] + return add_infos,delete_infos + + def _check_master_add_rules(self,add_infos): + ''' + check master add pkgs rule + ''' + log.info("add pkgs from check") + error_infos = {} + error_flag = False + for branch,pkgs in add_infos.items(): + log.info('branch:{}'.format(branch)) + log.info('pkgs:{}'.format(pkgs)) + if branch == 'openEuler-Factory': + branch = branch.replace("-",":") + for pkg in pkgs: + if pkg['obs_to'] != branch: + error_flag = True + log.error("pkg name:{2} Wrong obs_to: in !!!".format(pkg['obs_to'],branch,pkg['name'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + else: + for pkg in pkgs: + if 'Multi-Version' in branch: + branch = branch.replace("_",":") + from_result = self._find_master_meta_path(pkg,ctype='multi-from') + else: + branch = branch.replace("-",":") + from_result = self._find_master_meta_path(pkg) + if pkg['obs_to'] != branch: + error_flag = True + log.error("pkg name:{2} Wrong obs_to: in !!!".format(pkg['obs_to'],branch,pkg['name'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if from_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + + def _find_master_meta_path(self,pkg,ctype='from'): + ''' + find obs_form in obs_meta path + ''' + if ctype == 'from': + pkg_from_path = os.path.join(self.meta_path, 'master', pkg['obs_from'], pkg['name']) + if not os.path.exists(pkg_from_path): + yaml_key = os.path.join('master',pkg['obs_from'], pkg['name']) + log.error("The {0} not exist in obs_meta".format(yaml_key)) + return True + return False + elif ctype == 'multi-from': + if 'Multi-Version' in pkg['source_dir']: + dir_name = '{}/{}'.format(pkg['source_dir'],pkg['obs_from']) + pkg_from_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) + yaml_key = os.path.join('multi_version', dir_name, pkg['name']) + else: + pkg_from_path = os.path.join(self.meta_path, pkg['source_dir'], pkg['obs_from'], pkg['name']) + yaml_key = os.path.join(pkg['source_dir'], pkg['obs_from'], pkg['name']) + if not os.path.exists(pkg_from_path): + log.error("The {0} not exist in obs_meta".format(yaml_key)) + return True + return False + elif ctype == 'multi-to': + if 'Multi-Version' in pkg['destination_dir']: + dir_name = '{}/{}'.format(pkg['destination_dir'],pkg['obs_to']) + pkg_to_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) + yaml_key = os.path.join('multi_version', dir_name, pkg['name']) + else: + pkg_to_path = os.path.join(self.meta_path, pkg['destination_dir'], pkg['obs_to'], pkg['name']) + yaml_key = os.path.join(pkg['destination_dir'], pkg['obs_to'], pkg['name']) + if not os.path.exists(pkg_to_path): + log.error("The pkg {0} you want delete not exist in obs_meta".format(yaml_key)) + return True + return False + else: + pkg_to_path = os.path.join(self.meta_path, 'master', pkg['obs_to'], pkg['name']) + yaml_key = os.path.join('master', pkg['obs_to'], pkg['name']) + if not os.path.exists(pkg_to_path): + log.error("The pkg {0} you want delete not exist in obs_meta".format(yaml_key)) + return True + return False + + def _check_master_delete_rules(self,delete_infos): + ''' + check master delete pkgs rule + ''' + log.info("delete pkgs check") + error_infos = {} + error_flag = False + for branch,pkgs in delete_infos.items(): + log.info('branch:{}'.format(branch)) + log.info('pkgs:{}'.format(pkgs)) + if branch == 'openEuler-Factory': + branch = branch.replace("-",":") + for pkg in pkgs: + to_result = self._find_master_meta_path(pkg,ctype='to') + else: + # branch = branch.replace("-",":") + for pkg in pkgs: + if 'Multi-Version' in branch: + branch = branch.replace("_",":") + to_result = self._find_master_meta_path(pkg,ctype='multi-to') + else: + branch = branch.replace("-",":") + to_result = self._find_master_meta_path(pkg,ctype='to') + if to_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + def _check_master_date_rules(self,infos): + log.info("add pkgs date check") + error_infos = {} + error_flag = False + date = datetime.date.today() + today = date.day + for branch,pkgs in infos.items(): + for pkg in pkgs: + yaml_date = int(pkg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error(pkg) + log.error("Wrong Date: !!!".format(pkg['date'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + + def _check_pkg_from_and_date(self, meta_path, change_info,vtype='add'): + """ + check add or delete pkg date and obs_from or obs_to + """ + error_flag = False + date = datetime.date.today() + today = date.day + for branch,change in change_info.items(): + if vtype == 'add': + for msg in change: + log.info("{0} pkg_from and date check".format(msg['name'])) + yaml_date = int(msg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error(msg) + log.error("Wrong Date: !!!".format(msg['date'])) + msg_path = os.path.join(meta_path, msg['source_dir'],msg['obs_from'], msg['name']) + yaml_key = os.path.join(msg['source_dir'],msg['obs_from'], msg['name']) + if not os.path.exists(msg_path): + log.error("The {0} not exist in obs_meta".format(yaml_key)) + error_flag = True + else: + pkgs = [msg['name'] for msg in change] + branch_dir = os.path.join(meta_path,branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path: + pkgs.remove(name) + if pkgs: + log.error("The {0} not exist in obs_meta dir {1}".format(pkgs,branch)) + error_flag = True + return error_flag + + def _check_key_in_yaml_new(self, change_info): + """ + check the key in your yaml compliance with rules + """ + error_flag = "" + # keylist = ['branch_from', 'obs_from', 'name', 'branch_to', 'obs_to', 'date', 'change_pr'] + keylist = ['source_dir', 'obs_from', 'name', 'destination_dir', 'obs_to', 'date'] + for branch,info in change_info.items(): + if info: + log.info("branch:{0},pkgs:{1} key check".format(branch,info)) + for msg in info: + if len(msg.keys()) == 7 or len(msg.keys()) == 6: + for key in msg.keys(): + if key not in keylist: + error_flag = True + log.error(msg) + log.error("ERROR:<<<<<<{0}:>>>>>> should not in there".format(key)) + else: + error_flag = True + log.error("Please check {0}".format(msg)) + if error_flag: + raise SystemExit("ERROR: Please ensure the following key values in your yaml") + + def _ensure_delete_infos(self,del_old_msg,del_new_msg): + ''' + check new version delete pkgs + ''' + del_new_pkg = {} + del_old_pkg = {} + ensure_delete_pkg = {} + for branch,del_new_msgs in del_new_msg.items(): + del_change_pkgs = [] + del_new_pkg[branch] = [new['name'] for new in del_new_msgs] + del_old_pkg[branch] = [old['name'] for old in del_old_msg[branch]] + delete_pkgs = list(set(del_new_pkg[branch]).difference(set(del_old_pkg[branch]))) + if delete_pkgs: + for del_new in del_new_msgs: + if del_new['name'] in delete_pkgs: + del_change_pkgs.append(del_new) + ensure_delete_pkg[branch] = del_change_pkgs + for branch,branch_del_pkgs in ensure_delete_pkg.items(): + if branch_del_pkgs: + log.info("branch:{},delete pkgs:{}".format(branch,branch_del_pkgs)) + return ensure_delete_pkg + + + def _check_rpms_complete_and_repeat(self,old_msg,new_msg): + ''' + compare with old and new yaml msg, make sure package exist + ''' + old_pkg = {} + new_pkg = {} + same_pkg = {} + error_pkg = {} + change_infos = {} + error_pkg_flag = False + same_pkg_flag = False + log.info("new version rpms exists and repeat check") + for branch,new_msgs in new_msg.items(): + old_pkg[branch] = [] + new_pkg[branch] = [] + same_pkg[branch] = [] + change_pkgs = [] + old_msgs = old_msg[branch] + for old in old_msgs: + old_pkg[branch].append(old['name']) + for new in new_msgs: + if new['name'] in new_pkg[branch]: + same_pkg_flag = True + same_pkg[branch].append(new['name']) + new_pkg[branch].append(new['name']) + error_branch_pkgs = list(set(old_pkg[branch]).difference(set(new_pkg[branch]))) + if error_branch_pkgs: + error_pkg[branch] = error_branch_pkgs + error_pkg_flag = True + add_pkgs = list(set(new_pkg[branch]).difference(set(old_pkg[branch]))) + for new in new_msgs: + if new['name'] in add_pkgs: + change_pkgs.append(new) + change_infos[branch] = change_pkgs + for branch,pkgs in change_infos.items(): + if pkgs: + log.info("branch:{},add pkgs:{}".format(branch,pkgs)) + if error_pkg_flag: + log.error("May be {0} should not be delete".format(error_pkg)) + raise SystemExit("ERROR: Please check your PR") + if same_pkg_flag: + log.error("The following {0} packages are duplicated in the YAML files".format(same_pkg)) + raise SystemExit("ERROR: Please check your PR") + return change_infos + + def _get_new_version_yaml_msg(self, yaml_path_list, manage_path,vtype='master'): + ''' + get new version yaml msg content + ''' + all_pack_msg = {} + all_del_msg = {} + for yaml_path in yaml_path_list: + result = {} + if vtype == 'master': + branch_infos = yaml_path.split('/') + branch = branch_infos[1] + else: + branch_infos = yaml_path.split('/') + branch = branch_infos[0] + file_path = os.path.join(manage_path, yaml_path) + if os.path.exists(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + yaml_packages = [pkg for pkg in result['packages']] + if 'delete' in branch_infos: + all_del_msg[branch] = yaml_packages + if all_pack_msg.get(branch,''): + full_packags = all_pack_msg[branch] + yaml_packages + all_pack_msg[branch] = full_packags + else: + all_pack_msg[branch] = yaml_packages + else: + all_pack_msg[branch] = [] + return all_pack_msg,all_del_msg + + def check_pckg_yaml(self): """ check the obs_from branch_from in pckg-mgmt.yaml """ change = self._get_repo_change_file('openeuler', 'release-management', self.manage_path) - change_file = self._parse_commit_file(change) - self._check_yaml_format(change_file, self.manage_path) + change_file,master_change_file,new_version_change_file = self._parse_commit_file(change) + all_change_files = [*change_file, *master_change_file, *new_version_change_file] + self._check_yaml_format(all_change_files, self.manage_path) all_yaml_msg = self._get_allkey_msg(change_file, self.manage_path) change_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) - old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path, True) - self._check_rpms_integrity(old_yaml_msg, change_yaml_msg, change_file) - change_msg_list = self._get_diff_msg(old_yaml_msg, change_yaml_msg, change_file) - log.info(len(change_msg_list)) - self._ensure_delete_tags(change_msg_list, old_yaml_msg, all_yaml_msg) - self._check_key_in_yaml(change_msg_list, change_file) - error_flag1 = self._check_pkg_from(self.meta_path, change_msg_list, change_file, all_yaml_msg) - error_flag2 = self._check_date_time(change_msg_list, change_file) - error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) - error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) - if error_flag1 or error_flag2 or error_flag3 or error_flag4: - raise SystemExit("Please check your commit") + all_master_yaml_msg = self._get_allkey_msg(master_change_file, self.manage_path) + master_change_yaml_msg,del_master_change_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + new_version_change_msg,del_new_version_change_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') + self._rollback_get_msg(self.manage_path) + old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) + old_master_yaml_msg,del_old_master_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + old_new_version_msg,del_old_new_version_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') + if master_change_file: + log.info(master_change_file) + add_infos,delete_infos= self._get_delete_and_add(old_master_yaml_msg,master_change_yaml_msg) + add_flag = self._check_master_add_rules(add_infos) + del_flag = self._check_master_delete_rules(delete_infos) + date_flag = self._check_master_date_rules(add_infos) + if add_flag or del_flag or date_flag: + raise SystemExit("Please check your commit") + if new_version_change_file: + log.info(new_version_change_file) + change_infos = self._check_rpms_complete_and_repeat(old_new_version_msg, new_version_change_msg) + change_delete_infos = self._ensure_delete_infos(del_old_new_version_msg,del_new_version_change_msg) + log.info("change pkgs yaml key check") + self._check_key_in_yaml_new(change_infos) + # self._check_key_in_yaml_new(change_delete_infos) + log.info("change pkgs and delete yaml obs_from and obs_to check") + error_flag_add = self._check_pkg_from_and_date(self.meta_path, change_infos) + error_flag_del = self._check_pkg_from_and_date(self.meta_path, change_delete_infos,vtype='delete') + if error_flag_add or error_flag_del: + raise SystemExit("Please check your commit") + if change_file: + log.info(change_file) + self._check_rpms_integrity(old_yaml_msg, change_yaml_msg, change_file) + change_msg_list = self._get_diff_msg(old_yaml_msg, change_yaml_msg, change_file) + self._ensure_delete_tags(change_msg_list, old_yaml_msg, all_yaml_msg) + self._check_key_in_yaml(change_msg_list, change_file) + error_flag1 = self._check_pkg_from(self.meta_path, change_msg_list, change_file, all_yaml_msg) #添加multi_version判断 + error_flag2 = self._check_date_time(change_msg_list, change_file) + error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) + error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) + if error_flag1 or error_flag2 or error_flag3 or error_flag4: + raise SystemExit("Please check your commit") if __name__ == "__main__": kw = {"branch":"master", - "gitee_user":"", - "gitee_pwd":"", - "pr_id":"", - "obs_meta_path":"***", - "release_management_path":"***"} + "gitee_user":"dongjie110", + "gitee_pwd":"Dj@6445240", + "pr_id":"46", + "obs_meta_path":"/home/0621/obs_meta", + "release_management_path":"/home/0621/release-management"} check = CheckReleaseManagement(**kw) - check.check_pckg_yaml() + check.check_pckg_yaml() \ No newline at end of file diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index e0e18d9ee9082e7512ae29268eb21a9ab833cda3..852aa72dc7afa4f28470d03e96f9634556de6761 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -18,9 +18,11 @@ Synchronize the obs_meta file according to the pckg-mgmt.yaml file """ import os +import re import sys import yaml import shutil +import datetime now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(os.path.join(now_path, "..")) from common.log_obs import log @@ -97,6 +99,36 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) + elif yaml_type == 'multi-new': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['source_dir'], + 'branch_to': pckg['destination_dir'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['destination_dir'] + msg.append(tmp) + elif yaml_type == 'multi-new-delete': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name']} + msg.append(tmp) else: for label in file_msg['packages']['everything']: for pckg in file_msg['packages']['everything'][label]: @@ -113,16 +145,16 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) return msg, del_msg, prj_pkg def _write_prj_meta_file(self, file_path, proj): @@ -141,7 +173,24 @@ class SyncPckgMgmt(object): con_proj = main_proj.replace(':', '_').lower() epol_repo_aarch64 = "" epol_repo_x86 = "" - file_msg = """ + # if 'openEuler:Testing' in proj:#Testing + if 'master' in file_path: + file_msg = """ + + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="standard_aarch64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> <title/> <description/> <person userid="Admin" role="maintainer"/> @@ -157,7 +206,7 @@ class SyncPckgMgmt(object): <arch>x86_64</arch> </repository> </project> -""".format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) f = open(file_path, "w") f.write(file_msg) f.close() @@ -192,8 +241,17 @@ class SyncPckgMgmt(object): """ add obs_meta packages _service file """ - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + success_pkg_name = '' + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) pkg_service_path = os.path.join(pkg_path, "_service") if tmp['branch_from'] == "master": branch = "openEuler" @@ -206,36 +264,137 @@ class SyncPckgMgmt(object): if os.system(cmd) == 0: cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.error("copy %s service file failed!" % tmp['pkgname']) - - def _add_prj_meta_pkgs_service(self, msg): + return success_pkg_name + + def _move_pkg_service(self,tmp): + success_pkg_name = '' + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + if not os.path.exists(pkg_path): + cmd = "mv %s %s" % (from_pkg_path,mv_to_path) + if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] + log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return success_pkg_name + + def _add_master_pkg_service(self,package): + """ + create and add master service to repo obs_meta + """ + filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) + if not os.path.exists(filepath): + os.makedirs(filepath) + file_msg = """<services> + <service name="tar_scm"> + <param name="scm">git</param> + <param name="url">git@gitee.com:src-openeuler/{}.git</param> + <param name="exclude">*</param> + <param name="extract">*</param> + <param name="revision">master</param> + </service> +</services>""".format(package) + try: + with open(os.path.join(filepath,'_service'),'w') as f: + f.write(file_msg) + log.info("add %s _service success!" % (filepath)) + except Exception as e: + print (e) + log.info("add %s _service failed!" % (filepath)) + + def _move_master_pkg_service(self, msg): + """ + copy branch master obs_meta packages _service file + """ + change_pkgs = [] + for tmp in msg: + if tmp['obs_to'] == 'openEuler:Factory': + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if not os.path.exists(pkg_path): + self._add_master_pkg_service(tmp['pkgname']) + change_pkgs.append(tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return change_pkgs + + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): + pkg_names = [] for tmp in msg: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to']: - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - self._add_pkg_service(tmp) + if 'multi_version' not in branch_infos: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'master' not in branch_infos: + if tmp['branch_from'] == tmp['branch_to']: + pkg_name = self._move_pkg_service(tmp) + else: + pkg_name = self._add_pkg_service(tmp) + if pkg_name: + pkg_names.append(pkg_name) + return pkg_names + def _del_pkg(self, tmp): """ delete obs_meta packages """ - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) if os.path.exists(pkg_path): shutil.rmtree(pkg_path) log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] + + def _del_pkg_new(self, pkgs, branch): + """ + find in branch dirs and delete pkg + """ + need_del_path = [] + correct_del_pkgs = [] + branch_dir = os.path.join(self.obs_meta_path,branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path: + correct_del_pkgs.append(name) + need_del_path.append(c_path) + for pkg_path in need_del_path: + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + log.info("delete %s succeed!" % (pkg_path)) + return correct_del_pkgs def _verify_meta_file(self, prj_pkg): """ @@ -243,19 +402,31 @@ class SyncPckgMgmt(object): """ for proj, pkg in prj_pkg.items(): if proj != "branch": - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) - need_del_pkg = set(meta_pkglist) - set(pkg) + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + if len(list(prj_pkg.keys())) >= 3: + need_del_pkg = [] + else: + need_del_pkg = set(meta_pkglist) - set(pkg) log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) - for del_pkg in need_del_pkg: - tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} - self._del_pkg(tmp) + if need_del_pkg: + for del_pkg in need_del_pkg: + tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} + self._del_pkg(tmp) + return list(need_del_pkg) - def _push_code(self): + def _push_code(self,repo): """ push code to gitee repo """ - if os.path.exists(self.obs_meta_path): - os.chdir(self.obs_meta_path) + if os.path.exists(repo): + os.chdir(repo) cmd = "git status -s" if os.popen(cmd).read(): cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" @@ -272,38 +443,107 @@ class SyncPckgMgmt(object): log.info("No change, nothing to commit!") return "nothing to push" else: - log.error("%s not exist!" % self.obs_meta_path) + log.error("%s not exist!" % repo) return -1 + def _write_release_yaml(self,change_pkgs,branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs and len(change_pkgs) < 50: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + else: + pull_request = commitid + log.info("ignore write release change yaml file success") + + def sync_yaml_meta(self): """ integration of functions """ change_file = self._get_change_file() + master_change_file = [] yaml_dict = {} + master_prj_pkg = [] for line in change_file: log.info("line:%s" % line) name = list(line.split())[1] - branch = name.split('/')[0] file_path = os.path.join(self.release_management_path, name) yaml_dict = self._get_yaml_file_msg(file_path) - if not yaml_dict: - log.info("%s file content is empty!" % name) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = name.split('/')[1] + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + master_prj_pkg.append(prj_pkg) + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + move_change_pkgs = self._move_master_pkg_service(msg) + self._write_release_yaml(move_change_pkgs,branch_infos[0]) + elif 'multi_version' in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = pkg_names + del_change_pkgs + complete_path = os.path.join(branch_infos[0],branch_infos[1]) + self._write_release_yaml(change_pkgs,complete_path) else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg) + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + # msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + if 'delete' not in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + self._write_release_yaml(pkg_names,branch_infos[0]) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") + pkgs = [tmp['pkgname'] for tmp in msg] + del_change_pkgs = self._del_pkg_new(pkgs,branch_infos[0]) + self._write_release_yaml(del_change_pkgs,branch_infos[0]) else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg) - for tmp in del_msg: - self._del_pkg(tmp) - self._verify_meta_file(prj_pkg) - ret = self._push_code() + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + for tmp in del_msg: + self._del_pkg(tmp) + del_change_pkgs = self._verify_meta_file(prj_pkg) + if master_prj_pkg: + for prj_pkg in master_prj_pkg: + master_del_pkgs = self._verify_meta_file(prj_pkg) + self._write_release_yaml(master_del_pkgs,'master') + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) return ret if __name__ == "__main__": - kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} mgmt = SyncPckgMgmt(**kw) - mgmt.sync_yaml_meta() + mgmt.sync_yaml_meta() \ No newline at end of file