From e225fb1791ecc16a6d3311b8e9271b80c5620a37 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 31 May 2022 17:39:32 +0800 Subject: [PATCH 01/13] =?UTF-8?q?1.release=5Fmanagement=E9=97=A8=E7=A6=81?= =?UTF-8?q?=E5=85=BC=E5=AE=B9=E6=96=B0=E6=96=B9=E6=A1=88=E7=9B=AE=E5=BD=95?= =?UTF-8?q?=E7=BB=93=E6=9E=84=202.=E5=A2=9E=E5=8A=A0master/multi-version?= =?UTF-8?q?=E7=9B=AE=E5=BD=95=E9=97=A8=E7=A6=81=E6=A3=80=E6=9F=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/check_release_management.py | 253 ++++++++++++++++++++++++++++--- 1 file changed, 232 insertions(+), 21 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 50f5877..e8b4504 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -118,19 +118,38 @@ class CheckReleaseManagement(object): get the change file for latest commit """ new_file_path = [] + new_versin_file_path = [] + master_new_file_path = [] for line in change_file: log.info("line:%s" % line) log_list = list(line.split()) temp_log_type = log_list[0] if len(log_list) == 3: if "pckg-mgmt.yaml" in log_list[2]: - new_file_path.append(log_list[2]) + if 'master' in log_list[2]: + master_new_file_path.append(log_list[2]) + elif 'multi-version' in log_list[2]: + new_file_path.append(log_list[2]) + else: + branch_infos = log_list[2].split('/') + if len(branch_infos) == 3: + new_versin_file_path.append(log_list[2]) + else: + new_file_path.append(log_list[2]) elif len(log_list) == 2: if temp_log_type != "D" and "pckg-mgmt.yaml" in log_list[1]: - new_file_path.append(log_list[1]) - if new_file_path: - log.info(new_file_path) - return new_file_path + if 'master' in log_list[1]: + master_new_file_path.append(log_list[1]) + elif 'multi-version' in log_list[1]: + new_file_path.append(log_list[1]) + else: + branch_infos = log_list[1].split('/') + if len(branch_infos) == 3: + new_versin_file_path.append(log_list[1]) + else: + new_file_path.append(log_list[1]) + if new_file_path or master_new_file_path or new_versin_file_path: + return new_file_path,master_new_file_path,new_versin_file_path else: log.info("There are no file need to check!!!") sys.exit() @@ -246,8 +265,11 @@ class CheckReleaseManagement(object): delete_tag = self._check_delete_tag(msg, yaml_all_msg[change]) if delete_tag: continue - msg_path = os.path.join(meta_path, msg['branch_from'], - msg['obs_from'], msg['name']) + if 'multi-version' in change: + msg_path = os.path.join(meta_path, 'multi_version', msg['branch_from'], msg['obs_from'], msg['name']) + else: + msg_path = os.path.join(meta_path, msg['branch_from'], + msg['obs_from'], msg['name']) if not os.path.exists(msg_path): yaml_key = os.path.join(msg['branch_from'], msg['obs_from'], msg['name']) @@ -404,28 +426,217 @@ class CheckReleaseManagement(object): log.error("Check the delete group in the {0}!!!".format(info_dict)) raise SystemExit("ERROR:Please check your PR") + def _parse_msgs(self,nowmsg): + pkgs = {} + for branch,msg in nowmsg.items(): + if branch not in ['openEuler-Factory']: + branch = 'openEuler-Other' + for line in msg: + if pkgs.get(branch,''): + pkgs[branch].append(line['name']) + else: + pkgs[branch] = [line['name']] + return pkgs + + def _check_master_rpms_rule(self,old_msg,new_msg): + change_msgs = {'openEuler-Factory':{},'openEuler-Other':{}} + new_pkgs = self._parse_msgs(new_msg) + old_pkgs = self._parse_msgs(old_msg) + for branch,msg in new_pkgs.items(): + delete = list(set(old_pkgs[branch]).difference(set(new_pkgs[branch]))) + add = list(set(new_pkgs[branch]).difference(set(old_pkgs[branch]))) + change_msgs[branch]['add'] = add + change_msgs[branch]['del'] = delete + openEuler_Factory = change_msgs.get('openEuler-Factory',{'add':[],'del':[]}) + openEuler_Other = change_msgs.get('openEuler-Other',{'add':[],'del':[]}) + error_other_add = list(set(openEuler_Other['add']).difference(set(openEuler_Factory['del']))) + correct_other_add = list(set(openEuler_Other['add']).intersection(set(openEuler_Factory['del']))) + error_fac_add = list(set(openEuler_Factory['add']).intersection(set(openEuler_Other['del']))) + correct_fac_add = list(set(openEuler_Factory['add']).difference(set(openEuler_Other['del']))) + if error_other_add or error_fac_add: + log.error("ERROR:as follow pkgs add or move rule error {}/{}".format(error_other_add,error_fac_add)) + raise SystemExit("ERROR: Please check your yaml") + if correct_fac_add or correct_other_add: + log.info("as follow pkgs add or move correct {}/{}".format(correct_fac_add,correct_other_add)) + + + def _check_pkg_from_and_date(self, meta_path, change_info): + """ + Detects the existence of file contents + """ + error_flag = False + date = datetime.date.today() + today = date.day + for branch,change in change_info.items(): + for msg in change: + log.info("{0} pkg_from and date check".format(msg['name'])) + msg_path = os.path.join(meta_path, msg['branch_from'], + msg['obs_from'], msg['name']) + yaml_date = int(msg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error(msg) + log.error("Wrong Date: !!!".format(msg['date'])) + if not os.path.exists(msg_path): + yaml_key = os.path.join(msg['branch_from'], + msg['obs_from'], msg['name']) + log.error("The {0} not exist in obs_meta".format(yaml_key)) + error_flag = True + return error_flag + + def _check_key_in_yaml_new(self, change_info): + """ + check the key in your yaml compliance with rules + """ + error_flag = "" + keylist = ['branch_from', 'obs_from', 'name', 'branch_to', 'obs_to', 'date', 'change_pr'] + log.info("{0} key check".format(change_info)) + for branch,info in change_info.items(): + for msg in info: + if len(msg.keys()) == 7 or len(msg.keys()) == 6: + for key in msg.keys(): + if key not in keylist: + error_flag = True + log.error(msg) + log.error("ERROR:<<<<<<{0}:>>>>>> should not in there".format(key)) + else: + error_flag = True + log.error("Please check {0}".format(msg)) + if error_flag: + raise SystemExit("ERROR: Please ensure the following key values in your yaml") + + def _ensure_delete_infos(self,del_old_msg,del_new_msg): + ''' + check new version delete pkgs + ''' + del_new_pkg = {} + del_old_pkg = {} + ensure_delete_pkg = {} + for branch,del_new_msgs in del_new_msg.items(): + del_new_pkg[branch] = [new['name'] for new in del_new_msgs] + del_old_pkg[branch] = [old['name'] for old in del_old_msg[branch]] + delete_pkgs = list(set(del_new_pkg[branch]).difference(set(del_old_pkg[branch]))) + if delete_pkgs: + ensure_delete_pkg[branch] = delete_pkgs + log.info("delete infos:{} {}".format(branch,delete_pkgs)) + + + def _check_rpms_complete_and_repeat(self,old_msg,new_msg): + ''' + compare with old and new yaml msg, make sure package exist + ''' + old_pkg = {} + new_pkg = {} + same_pkg = {} + error_pkg = {} + change_infos = {} + error_pkg_flag = False + same_pkg_flag = False + log.info("new version rpms exists and repeat check") + for branch,new_msgs in new_msg.items(): + old_pkg[branch] = [] + new_pkg[branch] = [] + same_pkg[branch] = [] + change_pkgs = [] + old_msgs = old_msg[branch] + for old in old_msgs: + old_pkg[branch].append(old['name']) + for new in new_msgs: + if new['name'] in new_pkg[branch]: + same_pkg_flag = True + same_pkg[branch].append(new['name']) + new_pkg[branch].append(new['name']) + error_branch_pkgs = list(set(old_pkg[branch]).difference(set(new_pkg[branch]))) + if error_branch_pkgs: + error_pkg[branch] = error_branch_pkgs + error_pkg_flag = True + add_pkgs = list(set(new_pkg[branch]).difference(set(old_pkg[branch]))) + for new in new_msgs: + if new['name'] in add_pkgs: + change_pkgs.append(new) + change_infos[branch] = change_pkgs + log.info("change infos:{}".format(change_infos)) + if error_pkg_flag: + log.error("May be {0} should not be delete".format(error_pkg)) + raise SystemExit("ERROR: Please check your PR") + if same_pkg_flag: + log.error("The following {0} packages are duplicated in the YAML files".format(same_pkg)) + raise SystemExit("ERROR: Please check your PR") + return change_infos + + def _get_new_version_yaml_msg(self, yaml_path_list, manage_path,vtype='master'): + ''' + get new version yaml msg content + ''' + all_pack_msg = {} + all_del_msg = {} + all_path_msg = {} + for yaml_path in yaml_path_list: + if vtype == 'master': + branch_infos = yaml_path.split('/') + branch = branch_infos[1] + else: + branch_infos = yaml_path.split('/') + branch = yaml_path.split('/')[0] + file_path = os.path.join(manage_path, yaml_path) + if os.path.exists(file_path): + with open(file_path, 'r', encoding='utf-8') as f: + result = yaml.load(f, Loader=yaml.FullLoader) + all_path_msg[yaml_path] = result['packages'] + if 'delete' in branch_infos: + all_del_msg[branch] = result['packages'] + if all_pack_msg.get(branch,''): + all_pack_msg[branch].extend(result['packages']) + else: + all_pack_msg[branch] = result['packages'] + else: + all_pack_msg[branch] = [] + return all_pack_msg,all_del_msg + + def check_pckg_yaml(self): """ check the obs_from branch_from in pckg-mgmt.yaml """ change = self._get_repo_change_file('openeuler', 'release-management', self.manage_path) - change_file = self._parse_commit_file(change) - self._check_yaml_format(change_file, self.manage_path) + change_file,master_change_file,new_version_change_file = self._parse_commit_file(change) + all_change_files = [*change_file, *master_change_file, *new_version_change_file] + self._check_yaml_format(all_change_files, self.manage_path) all_yaml_msg = self._get_allkey_msg(change_file, self.manage_path) change_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) - old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path, True) - self._check_rpms_integrity(old_yaml_msg, change_yaml_msg, change_file) - change_msg_list = self._get_diff_msg(old_yaml_msg, change_yaml_msg, change_file) - log.info(len(change_msg_list)) - self._ensure_delete_tags(change_msg_list, old_yaml_msg, all_yaml_msg) - self._check_key_in_yaml(change_msg_list, change_file) - error_flag1 = self._check_pkg_from(self.meta_path, change_msg_list, change_file, all_yaml_msg) - error_flag2 = self._check_date_time(change_msg_list, change_file) - error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) - error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) - if error_flag1 or error_flag2 or error_flag3 or error_flag4: - raise SystemExit("Please check your commit") + all_master_yaml_msg = self._get_allkey_msg(master_change_file, self.manage_path) + master_change_yaml_msg,del_master_change_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + # all_new_version_msg = self._get_allkey_msg(new_version_change_file, self.manage_path) + new_version_change_msg,del_new_version_change_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') + self._rollback_get_msg(self.manage_path) + old_master_yaml_msg,del_old_master_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) + old_new_version_msg,del_old_new_version_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') + old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) + if master_change_file: + log.info(master_change_file) + self._check_master_rpms_rule(old_master_yaml_msg,master_change_yaml_msg) + if new_version_change_file: + # check new version here + log.info(new_version_change_file) + change_infos = self._check_rpms_complete_and_repeat(old_new_version_msg, new_version_change_msg) + change_delete_infos = self._ensure_delete_infos(del_old_new_version_msg,del_new_version_change_msg) + self._check_key_in_yaml_new(change_infos) + error_flag = self._check_pkg_from_and_date(self.meta_path, change_infos) + if error_flag: + raise SystemExit("Please check your commit") + if change_file: + log.info(change_file) + self._check_rpms_integrity(old_yaml_msg, change_yaml_msg, change_file) + change_msg_list = self._get_diff_msg(old_yaml_msg, change_yaml_msg, change_file) + self._ensure_delete_tags(change_msg_list, old_yaml_msg, all_yaml_msg) + self._check_key_in_yaml(change_msg_list, change_file) + error_flag1 = self._check_pkg_from(self.meta_path, change_msg_list, change_file, all_yaml_msg) #添加multi_version判断 + error_flag2 = self._check_date_time(change_msg_list, change_file) + error_flag3 = self._check_same_pckg(change_file, change_yaml_msg) + error_flag4 = self._check_branch_msg(change_msg_list, change_file, self.manage_path) + if error_flag1 or error_flag2 or error_flag3 or error_flag4: + raise SystemExit("Please check your commit") if __name__ == "__main__": kw = {"branch":"master", -- Gitee From 3e8a06ec25def37252aa96e835ae8fbc12788c29 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 25 Jun 2022 18:15:42 +0800 Subject: [PATCH 02/13] 0625 modify sync and check release --- config/config.ini | 20 ++- core/check_release_management.py | 182 ++++++++++++++++----- core/sync_pckg_mgmt.py | 271 +++++++++++++++++++++++++------ 3 files changed, 378 insertions(+), 95 deletions(-) diff --git a/config/config.ini b/config/config.ini index 5ae97f3..09846d7 100644 --- a/config/config.ini +++ b/config/config.ini @@ -30,7 +30,7 @@ name = ci_check build [package_info_file] name = package_info.csv [branch_proj] -master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras +master = openEuler:Factory openEuler:Mainline openEuler:Epol openEuler:Extras openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Epol openEuler:Erlang openEuler:Factory openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Mainline openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby openEuler-20.03-LTS = openEuler:20.03:LTS openEuler:20.03:LTS:Epol openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1 openEuler:20.03:LTS:SP1:Epol openEuler:20.03:LTS:SP1:Extras openEuler-20.03-LTS-Next = openEuler:20.03:LTS:Next openEuler:20.03:LTS:Next:Epol openEuler:20.03:LTS:Next:Extras @@ -65,6 +65,22 @@ obs_pkg_rpms = https://gitee.com/unsunghero/obs_pkg_rpms openEuler-Mainline = openEuler:selfbuild:BaseOS/mainline_standard_aarch64 openEuler:selfbuild:BaseOS/mainline_standard_x86_64 openEuler-Epol = openEuler:selfbuild:BaseOS/epol_aarch64 openEuler:selfbuild:BaseOS/epol_x86_64 openEuler-Factory = openEuler:selfbuild:BaseOS/factory_aarch64 openEuler:selfbuild:BaseOS/factory_x86_64 +openEuler-BaseTools = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-C = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Common_Languages_Dependent_Tools = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Erlang = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Golang = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Java = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-KernelSpace = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Lua = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Meson = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-MultiLanguage = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Nodejs = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Ocaml = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Perl = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Python = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Qt = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 +openEuler-Ruby = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 openEuler-20.03-LTS = openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_aarch64 openEuler:20.03:LTS:selfbuild:BaseOS/openeuler_20.03_lts_x86_64 openEuler-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_x86_64 openEuler-20.03-LTS-SP1-Epol = openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_aarch64 openEuler:20.03:LTS:SP1:selfbuild:BaseOS/openeuler_20.03_lts_sp1_epol_x86_64 @@ -105,4 +121,4 @@ path = /srv/obs/build [obs_ignore_package] name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd mingw-crt mingw-gcc mingw-wine-gecko wine wine-mono autotune igh-ethercat-xenomai obs_meta wrf risc-v-kernel dde-dock dde-network-utils rubik gcc-cross opencv [obs_include_project] -name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely \ No newline at end of file +name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler:BaseTools openEuler:C openEuler:Common_Languages_Dependent_Tools openEuler:Epol openEuler:Erlang openEuler:Factory openEuler:Golang openEuler:Java openEuler:KernelSpace openEuler:Lua openEuler:Mainline openEuler:Meson openEuler:MultiLanguage openEuler:Nodejs openEuler:Ocaml openEuler:Testing:Perl openEuler:Python openEuler:Qt openEuler:Ruby \ No newline at end of file diff --git a/core/check_release_management.py b/core/check_release_management.py index e8b4504..47bca57 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -426,38 +426,133 @@ class CheckReleaseManagement(object): log.error("Check the delete group in the {0}!!!".format(info_dict)) raise SystemExit("ERROR:Please check your PR") - def _parse_msgs(self,nowmsg): - pkgs = {} - for branch,msg in nowmsg.items(): - if branch not in ['openEuler-Factory']: - branch = 'openEuler-Other' - for line in msg: - if pkgs.get(branch,''): - pkgs[branch].append(line['name']) - else: - pkgs[branch] = [line['name']] - return pkgs - - def _check_master_rpms_rule(self,old_msg,new_msg): - change_msgs = {'openEuler-Factory':{},'openEuler-Other':{}} - new_pkgs = self._parse_msgs(new_msg) - old_pkgs = self._parse_msgs(old_msg) - for branch,msg in new_pkgs.items(): - delete = list(set(old_pkgs[branch]).difference(set(new_pkgs[branch]))) - add = list(set(new_pkgs[branch]).difference(set(old_pkgs[branch]))) - change_msgs[branch]['add'] = add - change_msgs[branch]['del'] = delete - openEuler_Factory = change_msgs.get('openEuler-Factory',{'add':[],'del':[]}) - openEuler_Other = change_msgs.get('openEuler-Other',{'add':[],'del':[]}) - error_other_add = list(set(openEuler_Other['add']).difference(set(openEuler_Factory['del']))) - correct_other_add = list(set(openEuler_Other['add']).intersection(set(openEuler_Factory['del']))) - error_fac_add = list(set(openEuler_Factory['add']).intersection(set(openEuler_Other['del']))) - correct_fac_add = list(set(openEuler_Factory['add']).difference(set(openEuler_Other['del']))) - if error_other_add or error_fac_add: - log.error("ERROR:as follow pkgs add or move rule error {}/{}".format(error_other_add,error_fac_add)) - raise SystemExit("ERROR: Please check your yaml") - if correct_fac_add or correct_other_add: - log.info("as follow pkgs add or move correct {}/{}".format(correct_fac_add,correct_other_add)) + def _get_delete_and_add(self,old_msg,new_msg): + ''' + get master project add and delete pkgs by compare old and new + ''' + add_infos = {} + delete_infos = {} + for branch,old_pkgs in old_msg.items(): + new_pkgs = new_msg[branch] + old_pkgs_names = [info['name'] for info in old_pkgs] + new_pkgs_names = [info['name'] for info in new_pkgs] + del_names = list(set(old_pkgs_names).difference(set(new_pkgs_names))) + add_names = list(set(new_pkgs_names).difference(set(old_pkgs_names))) + if del_names: + for old in old_pkgs: + if old['name'] in del_names: + if delete_infos.get(branch,[]): + delete_infos[branch].append(old) + else: + delete_infos[branch] = [old] + if add_names: + for new in new_pkgs: + if new['name'] in add_names: + if add_infos.get(branch,[]): + add_infos[branch].append(new) + else: + add_infos[branch] = [new] + print (add_infos) + print (delete_infos) + return add_infos,delete_infos + + def _check_master_add_rules(self,add_infos): + ''' + check master add pkgs rule + ''' + log.info("add pkgs check") + error_infos = {} + error_flag = False + for branch,pkgs in add_infos.items(): + if branch == 'openEuler-Factory': + pass + else: + branch = branch.replace("-",":") + for pkg in pkgs: + if pkg['obs_to'] != branch: + error_flag = True + log.error("pkg name:{2} Wrong obs_to: in !!!".format(pkg['obs_to'],branch,pkg['name'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + from_result = self._find_master_meta_path(pkg) + if from_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + + def _find_master_meta_path(self,pkg,ctype='from'): + ''' + find obs_form in obs_meta path + ''' + if ctype == 'from': + pkg_from_path = os.path.join(self.meta_path, 'master', pkg['obs_from'], pkg['name']) + if not os.path.exists(pkg_from_path): + log.error("The {0} not exist in obs_meta".format(pkg_from_path)) + return True + return False + else: + pkg_to_path = os.path.join(self.meta_path, 'master', pkg['obs_to'], pkg['name']) + if not os.path.exists(pkg_to_path): + log.error("The pkg {0} you want delete not exist in obs_meta".format(pkg_from_path)) + return True + return False + + def _check_master_delete_rules(self,delete_infos): + ''' + check master delete pkgs rule + ''' + log.info("delete pkgs check") + error_infos = {} + error_flag = False + for branch,pkgs in delete_infos.items(): + if branch == 'openEuler-Factory': + pass + else: + branch = branch.replace("-",":") + for pkg in pkgs: + to_result = self._find_master_meta_path(pkg,'to') + if to_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag + + def _check_master_date_rules(self,infos): + log.info("add pkgs date check") + error_infos = {} + error_flag = False + date = datetime.date.today() + today = date.day + for branch,pkgs in infos.items(): + if branch == 'openEuler-Factory': + pass + else: + branch = branch.replace("-",":") + for pkg in pkgs: + yaml_date = int(pkg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error(pkg) + log.error("Wrong Date: !!!".format(pkg['date'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] + if error_infos: + log.error("some errors in your commit,please check: {}".format(error_infos)) + return error_flag def _check_pkg_from_and_date(self, meta_path, change_info): @@ -577,7 +672,7 @@ class CheckReleaseManagement(object): branch = branch_infos[1] else: branch_infos = yaml_path.split('/') - branch = yaml_path.split('/')[0] + branch = branch_infos[0] file_path = os.path.join(manage_path, yaml_path) if os.path.exists(file_path): with open(file_path, 'r', encoding='utf-8') as f: @@ -607,7 +702,6 @@ class CheckReleaseManagement(object): change_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) all_master_yaml_msg = self._get_allkey_msg(master_change_file, self.manage_path) master_change_yaml_msg,del_master_change_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) - # all_new_version_msg = self._get_allkey_msg(new_version_change_file, self.manage_path) new_version_change_msg,del_new_version_change_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') self._rollback_get_msg(self.manage_path) old_master_yaml_msg,del_old_master_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) @@ -615,7 +709,13 @@ class CheckReleaseManagement(object): old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) if master_change_file: log.info(master_change_file) - self._check_master_rpms_rule(old_master_yaml_msg,master_change_yaml_msg) + # self._check_master_rpms_rule(old_master_yaml_msg,master_change_yaml_msg) + add_infos,delete_infos= self._get_delete_and_add(old_master_yaml_msg,master_change_yaml_msg) + add_flag = self._check_master_add_rules(add_infos) + del_flag = self._check_master_delete_rules(delete_infos) + date_flag = self._check_master_date_rules(add_infos) + if add_flag or del_flag or date_flag: + raise SystemExit("Please check your commit") if new_version_change_file: # check new version here log.info(new_version_change_file) @@ -640,10 +740,10 @@ class CheckReleaseManagement(object): if __name__ == "__main__": kw = {"branch":"master", - "gitee_user":"", - "gitee_pwd":"", - "pr_id":"", - "obs_meta_path":"***", - "release_management_path":"***"} + "gitee_user":"dongjie110", + "gitee_pwd":"Dj@6445240", + "pr_id":"14", + "obs_meta_path":"/home/0621/obs_meta", + "release_management_path":"/home/0621/release-management"} check = CheckReleaseManagement(**kw) - check.check_pckg_yaml() + check.check_pckg_yaml() \ No newline at end of file diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index e0e18d9..07334db 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -18,9 +18,11 @@ Synchronize the obs_meta file according to the pckg-mgmt.yaml file """ import os +import re import sys import yaml import shutil +import datetime now_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(os.path.join(now_path, "..")) from common.log_obs import log @@ -97,6 +99,32 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) + elif yaml_type == 'master': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': 'master', + 'branch_to': 'master', 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = 'master' + msg.append(tmp) + elif yaml_type == 'multi-new': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + prj_pkg['branch'] = pckg['branch_to'] + msg.append(tmp) else: for label in file_msg['packages']['everything']: for pckg in file_msg['packages']['everything'][label]: @@ -113,16 +141,16 @@ class SyncPckgMgmt(object): prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) prj_pkg['branch'] = pckg['branch_to'] msg.append(tmp) - for pckg in file_msg['packages']['recycle']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], - 'obs_to': pckg['obs_to']} - prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - msg.append(tmp) - for pckg in file_msg['packages']['delete']: - tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], - 'obs_to': pckg['obs_to']} - del_msg.append(tmp) + for pckg in file_msg['packages']['recycle']: + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], + 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + 'obs_to': pckg['obs_to']} + prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) + msg.append(tmp) + for pckg in file_msg['packages']['delete']: + tmp = {'pkgname': pckg['name'], 'branch_to': pckg['branch_to'], + 'obs_to': pckg['obs_to']} + del_msg.append(tmp) return msg, del_msg, prj_pkg def _write_prj_meta_file(self, file_path, proj): @@ -141,7 +169,24 @@ class SyncPckgMgmt(object): con_proj = main_proj.replace(':', '_').lower() epol_repo_aarch64 = "" epol_repo_x86 = "" - file_msg = """ + # if 'openEuler:Testing' in proj:#Testing + if 'master' in file_path: + file_msg = """ + + <description/> + <person userid="Admin" role="maintainer"/> + <repository name="standard_aarch64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_aarch64"/> + <arch>aarch64</arch> + </repository> + <repository name="standard_x86_64" rebuild="direct"> + <path project="openEuler:selfbuild:BaseOS" repository="baseos_x86_64"/> + <arch>x86_64</arch> + </repository> +</project> + """.format(proj) + else: + file_msg = """<project name="{0}"> <title/> <description/> <person userid="Admin" role="maintainer"/> @@ -157,7 +202,7 @@ class SyncPckgMgmt(object): <arch>x86_64</arch> </repository> </project> -""".format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) + """.format(proj, main_proj, con_proj, epol_repo_aarch64, epol_repo_x86) f = open(file_path, "w") f.write(file_msg) f.close() @@ -192,8 +237,17 @@ class SyncPckgMgmt(object): """ add obs_meta packages _service file """ - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + success_pkg_name = '' + if 'Multi-Version' in tmp['branch_from']: + dir_name = '{}/{}'.format(tmp['branch_from'],tmp['obs_from']) + from_pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) pkg_service_path = os.path.join(pkg_path, "_service") if tmp['branch_from'] == "master": branch = "openEuler" @@ -206,36 +260,71 @@ class SyncPckgMgmt(object): if os.system(cmd) == 0: cmd = "sed -i 's/%s\//%s\//g' %s/_service" % (branch, tmp['branch_to'], pkg_path) if os.system(cmd) == 0: + success_pkg_name = tmp['pkgname'] log.info("add %s %s %s _service succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.info("add %s %s %s _service failed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) else: log.error("copy %s service file failed!" % tmp['pkgname']) - - def _add_prj_meta_pkgs_service(self, msg): + return success_pkg_name + + def _move_master_pkg_service(self, msg): + """ + copy branch master obs_meta packages _service file + """ + change_pkgs = [] for tmp in msg: - prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) - if not os.path.exists(prj_meta_br): - os.makedirs(prj_meta_br) - prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) - if not os.path.exists(prj_meta_path): - self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) - selfbuild_meta_path = os.path.join(prj_meta_br, - "%s:selfbuild:BaseOS" % tmp['obs_to']) - if not os.path.exists(selfbuild_meta_path): - if "Epol" not in tmp['obs_to']: - self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) - self._add_pkg_service(tmp) + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + return change_pkgs + + + def _add_prj_meta_pkgs_service(self, msg, branch_infos): + pkg_names = [] + for tmp in msg: + if 'multi_version' not in branch_infos: + prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" + % tmp['branch_to']) + if not os.path.exists(prj_meta_br): + os.makedirs(prj_meta_br) + prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) + if not os.path.exists(prj_meta_path): + self._write_prj_meta_file(prj_meta_path, tmp['obs_to']) + selfbuild_meta_path = os.path.join(prj_meta_br, + "%s:selfbuild:BaseOS" % tmp['obs_to']) + if not os.path.exists(selfbuild_meta_path): + if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing + self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) + if 'master' not in branch_infos: + pkg_name = self._add_pkg_service(tmp) + if pkg_name: + pkg_names.append(pkg_name) + return pkg_names + def _del_pkg(self, tmp): """ delete obs_meta packages """ - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if 'Multi-Version' in tmp['branch_to']: + dir_name = '{}/{}'.format(tmp['branch_to'],tmp['obs_to']) + pkg_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name, tmp['pkgname']) + else: + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) if os.path.exists(pkg_path): shutil.rmtree(pkg_path) log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) + return tmp['pkgname'] def _verify_meta_file(self, prj_pkg): """ @@ -243,19 +332,31 @@ class SyncPckgMgmt(object): """ for proj, pkg in prj_pkg.items(): if proj != "branch": - meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) - need_del_pkg = set(meta_pkglist) - set(pkg) + if 'Multi-Version' in prj_pkg['branch']: + dir_name = '{}/{}'.format(prj_pkg['branch'],proj) + meta_path = os.path.join(self.obs_meta_path, 'multi_version', dir_name) + meta_pkglist = os.listdir(meta_path) + if 'README.md' in meta_pkglist: + meta_pkglist.remove('README.md') + else: + meta_pkglist = os.listdir(os.path.join(self.obs_meta_path, prj_pkg['branch'], proj)) + if len(list(prj_pkg.keys())) >= 3: + need_del_pkg = [] + else: + need_del_pkg = set(meta_pkglist) - set(pkg) log.info("obs_meta %s %s redundant pkg:%s" % (prj_pkg['branch'], proj, list(need_del_pkg))) - for del_pkg in need_del_pkg: - tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} - self._del_pkg(tmp) + if need_del_pkg: + for del_pkg in need_del_pkg: + tmp = {'pkgname': del_pkg, 'branch_to': prj_pkg['branch'], 'obs_to': proj} + self._del_pkg(tmp) + return list(need_del_pkg) - def _push_code(self): + def _push_code(self,repo): """ push code to gitee repo """ - if os.path.exists(self.obs_meta_path): - os.chdir(self.obs_meta_path) + if os.path.exists(repo): + os.chdir(repo) cmd = "git status -s" if os.popen(cmd).read(): cmd = "git add -A && git commit -m \"synchronize with pckg-mgmt.yaml file contents\"" @@ -272,38 +373,104 @@ class SyncPckgMgmt(object): log.info("No change, nothing to commit!") return "nothing to push" else: - log.error("%s not exist!" % self.obs_meta_path) + log.error("%s not exist!" % repo) return -1 + def _write_release_yaml(self,change_pkgs,branch): + ''' + write change info to release_change.yaml + ''' + if change_pkgs: + change_str = " ".join(change_pkgs) + os.chdir(self.release_management_path) + commit_cmd = 'git rev-parse HEAD' + commitid = os.popen(commit_cmd).read().split('\n')[0] + content_cmd = "git log --oneline -1" + content = os.popen(content_cmd).read().split('\n')[0] + reg=re.compile(r"(?<=!)\d+") + match=reg.search(content) + if match: + pr_id = match.group(0) + pull_request = "https://gitee.com/openeuler/release-management/pulls/{}".format(pr_id) + datestr = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') + release_change_yaml = os.path.join(self.release_management_path, branch, "release-change.yaml") + with open(release_change_yaml) as file: + result = yaml.load(file, Loader=yaml.FullLoader) + change_dic = { + 'pr':pull_request, + 'description':content, + 'changed_packages':change_str, + 'date':datestr + } + result['release-change'].append(change_dic) + with open(release_change_yaml, "w", encoding='utf-8') as f: + yaml.dump(result, f, default_flow_style=False, sort_keys=False) + log.info("write release change yaml file success") + else: + pull_request = commitid + log.info("ignore write release change yaml file success") + + def sync_yaml_meta(self): """ integration of functions """ change_file = self._get_change_file() + master_change_file = [] yaml_dict = {} + master_prj_pkg = [] for line in change_file: log.info("line:%s" % line) name = list(line.split())[1] - branch = name.split('/')[0] file_path = os.path.join(self.release_management_path, name) yaml_dict = self._get_yaml_file_msg(file_path) - if not yaml_dict: - log.info("%s file content is empty!" % name) + branch_infos = name.split('/') + if 'master' in branch_infos: + branch = name.split('/')[1] + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "master") + master_prj_pkg.append(prj_pkg) + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + move_change_pkgs = self._move_master_pkg_service(msg) + self._write_release_yaml(move_change_pkgs,branch_infos[0]) + elif 'multi_version' in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + del_change_pkgs = self._verify_meta_file(prj_pkg) + change_pkgs = pkg_names + del_change_pkgs + complete_path = os.path.join(branch_infos[0],branch_infos[1]) + self._write_release_yaml(change_pkgs,complete_path) else: - if "everything" in yaml_dict['packages'].keys(): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") - self._add_prj_meta_pkgs_service(msg) + if not yaml_dict: + log.info("%s file content is empty!" % name) + elif isinstance(yaml_dict['packages'], list): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + if 'delete' not in branch_infos: + pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) + change_pkgs = pkg_names + self._write_release_yaml(change_pkgs,branch_infos[0]) + else: + for tmp in msg: + self._del_pkg(tmp) else: - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") - self._add_prj_meta_pkgs_service(msg) - for tmp in del_msg: - self._del_pkg(tmp) + if "everything" in yaml_dict['packages'].keys(): + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + else: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "old") + pkg_names = self._add_prj_meta_pkgs_service(msg, branch_infos) + for tmp in del_msg: + self._del_pkg(tmp) + del_change_pkgs = self._verify_meta_file(prj_pkg) + if master_prj_pkg: + for prj_pkg in master_prj_pkg: self._verify_meta_file(prj_pkg) - ret = self._push_code() + ret = self._push_code(self.obs_meta_path) + self._push_code(self.release_management_path) return ret if __name__ == "__main__": - kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + # kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2]} + kw = {'gitee_user':sys.argv[1], 'gitee_pwd':sys.argv[2],'obs_meta_path':sys.argv[3],'release_management_path':sys.argv[4]} mgmt = SyncPckgMgmt(**kw) - mgmt.sync_yaml_meta() + mgmt.sync_yaml_meta() \ No newline at end of file -- Gitee From 4283d103386cdd20d26cf1121df29c3ea66e5314 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 28 Jun 2022 16:16:07 +0800 Subject: [PATCH 03/13] dsd --- core/check_release_management.py | 146 ++++++++++++++++++++----------- core/sync_pckg_mgmt.py | 24 +++++ 2 files changed, 121 insertions(+), 49 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 47bca57..44fcad2 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -120,6 +120,7 @@ class CheckReleaseManagement(object): new_file_path = [] new_versin_file_path = [] master_new_file_path = [] + multi_version_file_path = [] for line in change_file: log.info("line:%s" % line) log_list = list(line.split()) @@ -128,8 +129,8 @@ class CheckReleaseManagement(object): if "pckg-mgmt.yaml" in log_list[2]: if 'master' in log_list[2]: master_new_file_path.append(log_list[2]) - elif 'multi-version' in log_list[2]: - new_file_path.append(log_list[2]) + elif 'multi_version' in log_list[2]: + master_new_file_path.append(log_list[2]) else: branch_infos = log_list[2].split('/') if len(branch_infos) == 3: @@ -140,8 +141,8 @@ class CheckReleaseManagement(object): if temp_log_type != "D" and "pckg-mgmt.yaml" in log_list[1]: if 'master' in log_list[1]: master_new_file_path.append(log_list[1]) - elif 'multi-version' in log_list[1]: - new_file_path.append(log_list[1]) + elif 'multi_version' in log_list[1]: + master_new_file_path.append(log_list[1]) else: branch_infos = log_list[1].split('/') if len(branch_infos) == 3: @@ -265,11 +266,7 @@ class CheckReleaseManagement(object): delete_tag = self._check_delete_tag(msg, yaml_all_msg[change]) if delete_tag: continue - if 'multi-version' in change: - msg_path = os.path.join(meta_path, 'multi_version', msg['branch_from'], msg['obs_from'], msg['name']) - else: - msg_path = os.path.join(meta_path, msg['branch_from'], - msg['obs_from'], msg['name']) + msg_path = os.path.join(meta_path, msg['branch_from'],msg['obs_from'], msg['name']) if not os.path.exists(msg_path): yaml_key = os.path.join(msg['branch_from'], msg['obs_from'], msg['name']) @@ -452,8 +449,6 @@ class CheckReleaseManagement(object): add_infos[branch].append(new) else: add_infos[branch] = [new] - print (add_infos) - print (delete_infos) return add_infos,delete_infos def _check_master_add_rules(self,add_infos): @@ -467,8 +462,14 @@ class CheckReleaseManagement(object): if branch == 'openEuler-Factory': pass else: - branch = branch.replace("-",":") + # branch = branch.replace("-",":") for pkg in pkgs: + if 'Multi-Version' in branch: + branch = branch.replace("_",":") + from_result = self._find_master_meta_path(pkg,ctype='multi-from') + else: + branch = branch.replace("-",":") + from_result = self._find_master_meta_path(pkg) if pkg['obs_to'] != branch: error_flag = True log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) @@ -476,7 +477,7 @@ class CheckReleaseManagement(object): error_infos[branch].append(pkg) else: error_infos[branch] = [pkg] - from_result = self._find_master_meta_path(pkg) + # from_result = self._find_master_meta_path(pkg) if from_result: error_flag = True if error_infos.get(branch,[]): @@ -495,13 +496,39 @@ class CheckReleaseManagement(object): if ctype == 'from': pkg_from_path = os.path.join(self.meta_path, 'master', pkg['obs_from'], pkg['name']) if not os.path.exists(pkg_from_path): - log.error("The {0} not exist in obs_meta".format(pkg_from_path)) + yaml_key = os.path.join('master',pkg['obs_from'], pkg['name']) + log.error("The {0} not exist in obs_meta".format(yaml_key)) + return True + return False + elif ctype == 'multi-from': + if 'Multi-Version' in pkg['branch_from']: + dir_name = '{}/{}'.format(pkg['branch_from'],pkg['obs_from']) + pkg_from_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) + yaml_key = os.path.join('multi_version', dir_name, pkg['name']) + else: + pkg_from_path = os.path.join(self.meta_path, pkg['branch_from'], pkg['obs_from'], pkg['name']) + yaml_key = os.path.join(pkg['branch_from'], pkg['obs_from'], pkg['name']) + if not os.path.exists(pkg_from_path): + log.error("The {0} not exist in obs_meta".format(yaml_key)) + return True + return False + elif ctype == 'multi-to': + if 'Multi-Version' in pkg['branch_to']: + dir_name = '{}/{}'.format(pkg['branch_to'],pkg['obs_to']) + pkg_to_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) + yaml_key = os.path.join('multi_version', dir_name, pkg['name']) + else: + pkg_to_path = os.path.join(self.meta_path, pkg['branch_to'], pkg['obs_to'], pkg['name']) + yaml_key = os.path.join(pkg['branch_to'], pkg['obs_to'], pkg['name']) + if not os.path.exists(pkg_to_path): + log.error("The pkg {0} you want delete not exist in obs_meta".format(yaml_key)) return True return False else: pkg_to_path = os.path.join(self.meta_path, 'master', pkg['obs_to'], pkg['name']) + yaml_key = os.path.join('master', pkg['obs_to'], pkg['name']) if not os.path.exists(pkg_to_path): - log.error("The pkg {0} you want delete not exist in obs_meta".format(pkg_from_path)) + log.error("The pkg {0} you want delete not exist in obs_meta".format(yaml_key)) return True return False @@ -516,9 +543,15 @@ class CheckReleaseManagement(object): if branch == 'openEuler-Factory': pass else: - branch = branch.replace("-",":") + # branch = branch.replace("-",":") for pkg in pkgs: - to_result = self._find_master_meta_path(pkg,'to') + if 'Multi-Version' in branch: + branch = branch.replace("_",":") + to_result = self._find_master_meta_path(pkg,ctype='multi-to') + else: + branch = branch.replace("-",":") + to_result = self._find_master_meta_path(pkg,ctype='to') + # to_result = self._find_master_meta_path(pkg,'to') if to_result: error_flag = True if error_infos.get(branch,[]): @@ -539,7 +572,7 @@ class CheckReleaseManagement(object): if branch == 'openEuler-Factory': pass else: - branch = branch.replace("-",":") + # branch = branch.replace("-",":") for pkg in pkgs: yaml_date = int(pkg['date'].split('-')[2]) if today != yaml_date: @@ -555,9 +588,9 @@ class CheckReleaseManagement(object): return error_flag - def _check_pkg_from_and_date(self, meta_path, change_info): + def _check_pkg_from_and_date(self, meta_path, change_info,vtype='add'): """ - Detects the existence of file contents + check add or delete pkg date and obs_from or obs_to """ error_flag = False date = datetime.date.today() @@ -565,16 +598,18 @@ class CheckReleaseManagement(object): for branch,change in change_info.items(): for msg in change: log.info("{0} pkg_from and date check".format(msg['name'])) - msg_path = os.path.join(meta_path, msg['branch_from'], - msg['obs_from'], msg['name']) yaml_date = int(msg['date'].split('-')[2]) if today != yaml_date: error_flag = True log.error(msg) log.error("Wrong Date: <date:{0}>!!!".format(msg['date'])) + if vtype == 'add': + msg_path = os.path.join(meta_path, msg['branch_from'],msg['obs_from'], msg['name']) + yaml_key = os.path.join(msg['branch_from'],msg['obs_from'], msg['name']) + else: + msg_path = os.path.join(meta_path, msg['branch_to'],msg['obs_to'], msg['name']) + yaml_key = os.path.join(msg['branch_to'],msg['obs_to'], msg['name']) if not os.path.exists(msg_path): - yaml_key = os.path.join(msg['branch_from'], - msg['obs_from'], msg['name']) log.error("The {0} not exist in obs_meta".format(yaml_key)) error_flag = True return error_flag @@ -585,18 +620,19 @@ class CheckReleaseManagement(object): """ error_flag = "" keylist = ['branch_from', 'obs_from', 'name', 'branch_to', 'obs_to', 'date', 'change_pr'] - log.info("{0} key check".format(change_info)) for branch,info in change_info.items(): - for msg in info: - if len(msg.keys()) == 7 or len(msg.keys()) == 6: - for key in msg.keys(): - if key not in keylist: - error_flag = True - log.error(msg) - log.error("ERROR:<<<<<<{0}:>>>>>> should not in there".format(key)) - else: - error_flag = True - log.error("Please check {0}".format(msg)) + if info: + log.info("branch:{0},pkgs:{1} key check".format(branch,info)) + for msg in info: + if len(msg.keys()) == 7 or len(msg.keys()) == 6: + for key in msg.keys(): + if key not in keylist: + error_flag = True + log.error(msg) + log.error("ERROR:<<<<<<{0}:>>>>>> should not in there".format(key)) + else: + error_flag = True + log.error("Please check {0}".format(msg)) if error_flag: raise SystemExit("ERROR: Please ensure the following key values in your yaml") @@ -608,12 +644,19 @@ class CheckReleaseManagement(object): del_old_pkg = {} ensure_delete_pkg = {} for branch,del_new_msgs in del_new_msg.items(): + del_change_pkgs = [] del_new_pkg[branch] = [new['name'] for new in del_new_msgs] del_old_pkg[branch] = [old['name'] for old in del_old_msg[branch]] delete_pkgs = list(set(del_new_pkg[branch]).difference(set(del_old_pkg[branch]))) if delete_pkgs: - ensure_delete_pkg[branch] = delete_pkgs - log.info("delete infos:{} {}".format(branch,delete_pkgs)) + for del_new in del_new_msgs: + if del_new['name'] in delete_pkgs: + del_change_pkgs.append(del_new) + ensure_delete_pkg[branch] = del_change_pkgs + for branch,branch_del_pkgs in ensure_delete_pkg.items(): + if branch_del_pkgs: + log.info("branch:{},delete pkgs:{}".format(branch,branch_del_pkgs)) + return ensure_delete_pkg def _check_rpms_complete_and_repeat(self,old_msg,new_msg): @@ -650,7 +693,9 @@ class CheckReleaseManagement(object): if new['name'] in add_pkgs: change_pkgs.append(new) change_infos[branch] = change_pkgs - log.info("change infos:{}".format(change_infos)) + for branch,pkgs in change_infos.items(): + if pkgs: + log.info("branch:{},add pkgs:{}".format(branch,pkgs)) if error_pkg_flag: log.error("May be {0} should not be delete".format(error_pkg)) raise SystemExit("ERROR: Please check your PR") @@ -665,8 +710,8 @@ class CheckReleaseManagement(object): ''' all_pack_msg = {} all_del_msg = {} - all_path_msg = {} for yaml_path in yaml_path_list: + result = {} if vtype == 'master': branch_infos = yaml_path.split('/') branch = branch_infos[1] @@ -677,13 +722,14 @@ class CheckReleaseManagement(object): if os.path.exists(file_path): with open(file_path, 'r', encoding='utf-8') as f: result = yaml.load(f, Loader=yaml.FullLoader) - all_path_msg[yaml_path] = result['packages'] + yaml_packages = [pkg for pkg in result['packages']] if 'delete' in branch_infos: - all_del_msg[branch] = result['packages'] + all_del_msg[branch] = yaml_packages if all_pack_msg.get(branch,''): - all_pack_msg[branch].extend(result['packages']) + full_packags = all_pack_msg[branch] + yaml_packages + all_pack_msg[branch] = full_packags else: - all_pack_msg[branch] = result['packages'] + all_pack_msg[branch] = yaml_packages else: all_pack_msg[branch] = [] return all_pack_msg,all_del_msg @@ -704,12 +750,11 @@ class CheckReleaseManagement(object): master_change_yaml_msg,del_master_change_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) new_version_change_msg,del_new_version_change_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') self._rollback_get_msg(self.manage_path) + old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) old_master_yaml_msg,del_old_master_yaml_msg = self._get_new_version_yaml_msg(master_change_file, self.manage_path) old_new_version_msg,del_old_new_version_msg = self._get_new_version_yaml_msg(new_version_change_file, self.manage_path,vtype='newversion') - old_yaml_msg = self._get_yaml_msg(change_file, self.manage_path) if master_change_file: log.info(master_change_file) - # self._check_master_rpms_rule(old_master_yaml_msg,master_change_yaml_msg) add_infos,delete_infos= self._get_delete_and_add(old_master_yaml_msg,master_change_yaml_msg) add_flag = self._check_master_add_rules(add_infos) del_flag = self._check_master_delete_rules(delete_infos) @@ -717,13 +762,16 @@ class CheckReleaseManagement(object): if add_flag or del_flag or date_flag: raise SystemExit("Please check your commit") if new_version_change_file: - # check new version here log.info(new_version_change_file) change_infos = self._check_rpms_complete_and_repeat(old_new_version_msg, new_version_change_msg) change_delete_infos = self._ensure_delete_infos(del_old_new_version_msg,del_new_version_change_msg) + log.info("change pkgs yaml key check") self._check_key_in_yaml_new(change_infos) - error_flag = self._check_pkg_from_and_date(self.meta_path, change_infos) - if error_flag: + self._check_key_in_yaml_new(change_delete_infos) + log.info("change pkgs and delete yaml obs_from and obs_to check") + error_flag_add = self._check_pkg_from_and_date(self.meta_path, change_infos) + error_flag_del = self._check_pkg_from_and_date(self.meta_path, change_delete_infos,vtype='delete') + if error_flag_add or error_flag_del: raise SystemExit("Please check your commit") if change_file: log.info(change_file) @@ -742,7 +790,7 @@ if __name__ == "__main__": kw = {"branch":"master", "gitee_user":"dongjie110", "gitee_pwd":"Dj@6445240", - "pr_id":"14", + "pr_id":"22", "obs_meta_path":"/home/0621/obs_meta", "release_management_path":"/home/0621/release-management"} check = CheckReleaseManagement(**kw) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 07334db..8bb0a8d 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -268,6 +268,30 @@ class SyncPckgMgmt(object): log.error("copy %s service file failed!" % tmp['pkgname']) return success_pkg_name + def _add_master_pkg_service(self,package): + """ + create and add master service to repo obs_meta + """ + filepath = os.path.join(self.obs_meta_path, 'master', 'openEuler:Factory', package) + if not os.path.exists(filepath): + os.makedirs(filepath) + file_msg = """<services> + <service name="tar_scm"> + <param name="scm">git</param> + <param name="url">git@gitee.com:src-openeuler/{}.git</param> + <param name="exclude">*</param> + <param name="extract">*</param> + <param name="revision">master</param> + </service> + </services>""".format(package) + try: + with open(os.path.join(filepath,'_service'),'w') as f: + f.write(file_msg) + log.info("add %s _service success!" % (filepath)) + except Exception as e: + print (e) + log.info("add %s _service failed!" % (filepath)) + def _move_master_pkg_service(self, msg): """ copy branch master obs_meta packages _service file -- Gitee From 2bc1f9e066df0e9fc1a2170cb43c9ef540677e64 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 28 Jun 2022 17:53:51 +0800 Subject: [PATCH 04/13] wdw --- core/check_release_management.py | 55 +++++++++++++++++--------------- core/sync_pckg_mgmt.py | 28 +++++++++------- 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 44fcad2..a562380 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -460,9 +460,16 @@ class CheckReleaseManagement(object): error_flag = False for branch,pkgs in add_infos.items(): if branch == 'openEuler-Factory': - pass + branch = branch.replace("-",":") + for pkg in pkgs: + if pkg['obs_to'] != branch: + error_flag = True + log.error("pkg name:{2} Wrong obs_to: <obs_to:{0}> in <project:{1}>!!!".format(pkg['obs_to'],branch,pkg['name'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] else: - # branch = branch.replace("-",":") for pkg in pkgs: if 'Multi-Version' in branch: branch = branch.replace("_",":") @@ -477,7 +484,6 @@ class CheckReleaseManagement(object): error_infos[branch].append(pkg) else: error_infos[branch] = [pkg] - # from_result = self._find_master_meta_path(pkg) if from_result: error_flag = True if error_infos.get(branch,[]): @@ -541,7 +547,8 @@ class CheckReleaseManagement(object): error_flag = False for branch,pkgs in delete_infos.items(): if branch == 'openEuler-Factory': - pass + branch = branch.replace("-",":") + to_result = self._find_master_meta_path(pkg,ctype='to') else: # branch = branch.replace("-",":") for pkg in pkgs: @@ -551,13 +558,12 @@ class CheckReleaseManagement(object): else: branch = branch.replace("-",":") to_result = self._find_master_meta_path(pkg,ctype='to') - # to_result = self._find_master_meta_path(pkg,'to') - if to_result: - error_flag = True - if error_infos.get(branch,[]): - error_infos[branch].append(pkg) - else: - error_infos[branch] = [pkg] + if to_result: + error_flag = True + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] if error_infos: log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag @@ -569,20 +575,19 @@ class CheckReleaseManagement(object): date = datetime.date.today() today = date.day for branch,pkgs in infos.items(): - if branch == 'openEuler-Factory': - pass - else: - # branch = branch.replace("-",":") - for pkg in pkgs: - yaml_date = int(pkg['date'].split('-')[2]) - if today != yaml_date: - error_flag = True - log.error(pkg) - log.error("Wrong Date: <date:{0}>!!!".format(pkg['date'])) - if error_infos.get(branch,[]): - error_infos[branch].append(pkg) - else: - error_infos[branch] = [pkg] + # if branch == 'openEuler-Factory': + # pass + # else: + for pkg in pkgs: + yaml_date = int(pkg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error(pkg) + log.error("Wrong Date: <date:{0}>!!!".format(pkg['date'])) + if error_infos.get(branch,[]): + error_infos[branch].append(pkg) + else: + error_infos[branch] = [pkg] if error_infos: log.error("some errors in your commit,please check: {}".format(error_infos)) return error_flag diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 8bb0a8d..ac123d2 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -298,18 +298,22 @@ class SyncPckgMgmt(object): """ change_pkgs = [] for tmp in msg: - from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) - pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) - pkg_service_path = os.path.join(pkg_path, "_service") - if not os.path.exists(pkg_path): - os.makedirs(pkg_path) - if not os.path.exists(pkg_service_path): - cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) - if os.system(cmd) == 0: - change_pkgs.append(tmp['pkgname']) - log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) - else: - log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + if tmp['obs_to'] == 'openEuler:Factory': + self._add_master_pkg_service(tmp['pkgname']) + change_pkgs.append(tmp['pkgname']) + else: + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + pkg_service_path = os.path.join(pkg_path, "_service") + if not os.path.exists(pkg_path): + os.makedirs(pkg_path) + if not os.path.exists(pkg_service_path): + cmd = "cp %s/_service %s/_service" % (from_pkg_path, pkg_path) + if os.system(cmd) == 0: + change_pkgs.append(tmp['pkgname']) + log.info("copy %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) + else: + log.info("copy %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) return change_pkgs -- Gitee From 3f2542565fdd1ccbcd9bbae3140d8c0f54bfacfc Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 28 Jun 2022 18:12:41 +0800 Subject: [PATCH 05/13] dwd --- core/check_release_management.py | 3 ++- core/sync_pckg_mgmt.py | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index a562380..cc10499 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -548,7 +548,8 @@ class CheckReleaseManagement(object): for branch,pkgs in delete_infos.items(): if branch == 'openEuler-Factory': branch = branch.replace("-",":") - to_result = self._find_master_meta_path(pkg,ctype='to') + for pkg in pkgs: + to_result = self._find_master_meta_path(pkg,ctype='to') else: # branch = branch.replace("-",":") for pkg in pkgs: diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index ac123d2..64d2d19 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -299,8 +299,10 @@ class SyncPckgMgmt(object): change_pkgs = [] for tmp in msg: if tmp['obs_to'] == 'openEuler:Factory': - self._add_master_pkg_service(tmp['pkgname']) - change_pkgs.append(tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + if not os.path.exists(pkg_path): + self._add_master_pkg_service(tmp['pkgname']) + change_pkgs.append(tmp['pkgname']) else: from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) -- Gitee From 00f8a48ddd454da2f014a7818af8b8eb4803d3df Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Tue, 28 Jun 2022 18:36:20 +0800 Subject: [PATCH 06/13] ww --- core/check_release_management.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index cc10499..d1a1459 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -455,10 +455,12 @@ class CheckReleaseManagement(object): ''' check master add pkgs rule ''' - log.info("add pkgs check") + log.info("add pkgs from check") error_infos = {} error_flag = False for branch,pkgs in add_infos.items(): + log.info('branch:{}'.format(branch)) + log.info('pkgs:{}'.format(pkgs)) if branch == 'openEuler-Factory': branch = branch.replace("-",":") for pkg in pkgs: @@ -546,6 +548,8 @@ class CheckReleaseManagement(object): error_infos = {} error_flag = False for branch,pkgs in delete_infos.items(): + log.info('branch:{}'.format(branch)) + log.info('pkgs:{}'.format(pkgs)) if branch == 'openEuler-Factory': branch = branch.replace("-",":") for pkg in pkgs: @@ -576,9 +580,6 @@ class CheckReleaseManagement(object): date = datetime.date.today() today = date.day for branch,pkgs in infos.items(): - # if branch == 'openEuler-Factory': - # pass - # else: for pkg in pkgs: yaml_date = int(pkg['date'].split('-')[2]) if today != yaml_date: -- Gitee From 19613808de6cfb900015e82926549b257b56980e Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Wed, 29 Jun 2022 16:44:36 +0800 Subject: [PATCH 07/13] 0629 --- core/check_release_management.py | 2 +- core/sync_pckg_mgmt.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index d1a1459..6fef612 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -797,7 +797,7 @@ if __name__ == "__main__": kw = {"branch":"master", "gitee_user":"dongjie110", "gitee_pwd":"Dj@6445240", - "pr_id":"22", + "pr_id":"35", "obs_meta_path":"/home/0621/obs_meta", "release_management_path":"/home/0621/release-management"} check = CheckReleaseManagement(**kw) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 64d2d19..2afcaee 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -410,7 +410,7 @@ class SyncPckgMgmt(object): ''' write change info to release_change.yaml ''' - if change_pkgs: + if change_pkgs and len(change_pkgs) < 50: change_str = " ".join(change_pkgs) os.chdir(self.release_management_path) commit_cmd = 'git rev-parse HEAD' -- Gitee From 270ddb64031eea8c03375e966650f57c861790fc Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 2 Jul 2022 09:15:42 +0800 Subject: [PATCH 08/13] ds --- core/check_release_management.py | 61 ++++++++++++++++++------------- core/sync_pckg_mgmt.py | 63 ++++++++++++++++++++++++++------ 2 files changed, 87 insertions(+), 37 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index 6fef612..fda73e2 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -509,25 +509,25 @@ class CheckReleaseManagement(object): return True return False elif ctype == 'multi-from': - if 'Multi-Version' in pkg['branch_from']: - dir_name = '{}/{}'.format(pkg['branch_from'],pkg['obs_from']) + if 'Multi-Version' in pkg['source_dir']: + dir_name = '{}/{}'.format(pkg['source_dir'],pkg['obs_from']) pkg_from_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) yaml_key = os.path.join('multi_version', dir_name, pkg['name']) else: - pkg_from_path = os.path.join(self.meta_path, pkg['branch_from'], pkg['obs_from'], pkg['name']) - yaml_key = os.path.join(pkg['branch_from'], pkg['obs_from'], pkg['name']) + pkg_from_path = os.path.join(self.meta_path, pkg['source_dir'], pkg['obs_from'], pkg['name']) + yaml_key = os.path.join(pkg['source_dir'], pkg['obs_from'], pkg['name']) if not os.path.exists(pkg_from_path): log.error("The {0} not exist in obs_meta".format(yaml_key)) return True return False elif ctype == 'multi-to': - if 'Multi-Version' in pkg['branch_to']: - dir_name = '{}/{}'.format(pkg['branch_to'],pkg['obs_to']) + if 'Multi-Version' in pkg['destination_dir']: + dir_name = '{}/{}'.format(pkg['destination_dir'],pkg['obs_to']) pkg_to_path = os.path.join(self.meta_path, 'multi_version', dir_name, pkg['name']) yaml_key = os.path.join('multi_version', dir_name, pkg['name']) else: - pkg_to_path = os.path.join(self.meta_path, pkg['branch_to'], pkg['obs_to'], pkg['name']) - yaml_key = os.path.join(pkg['branch_to'], pkg['obs_to'], pkg['name']) + pkg_to_path = os.path.join(self.meta_path, pkg['destination_dir'], pkg['obs_to'], pkg['name']) + yaml_key = os.path.join(pkg['destination_dir'], pkg['obs_to'], pkg['name']) if not os.path.exists(pkg_to_path): log.error("The pkg {0} you want delete not exist in obs_meta".format(yaml_key)) return True @@ -603,21 +603,29 @@ class CheckReleaseManagement(object): date = datetime.date.today() today = date.day for branch,change in change_info.items(): - for msg in change: - log.info("{0} pkg_from and date check".format(msg['name'])) - yaml_date = int(msg['date'].split('-')[2]) - if today != yaml_date: - error_flag = True - log.error(msg) - log.error("Wrong Date: <date:{0}>!!!".format(msg['date'])) - if vtype == 'add': - msg_path = os.path.join(meta_path, msg['branch_from'],msg['obs_from'], msg['name']) - yaml_key = os.path.join(msg['branch_from'],msg['obs_from'], msg['name']) - else: - msg_path = os.path.join(meta_path, msg['branch_to'],msg['obs_to'], msg['name']) - yaml_key = os.path.join(msg['branch_to'],msg['obs_to'], msg['name']) - if not os.path.exists(msg_path): - log.error("The {0} not exist in obs_meta".format(yaml_key)) + if vtype == 'add': + for msg in change: + log.info("{0} pkg_from and date check".format(msg['name'])) + yaml_date = int(msg['date'].split('-')[2]) + if today != yaml_date: + error_flag = True + log.error(msg) + log.error("Wrong Date: <date:{0}>!!!".format(msg['date'])) + msg_path = os.path.join(meta_path, msg['source_dir'],msg['obs_from'], msg['name']) + yaml_key = os.path.join(msg['source_dir'],msg['obs_from'], msg['name']) + if not os.path.exists(msg_path): + log.error("The {0} not exist in obs_meta".format(yaml_key)) + error_flag = True + else: + pkgs = [msg['name'] for msg in change] + branch_dir = os.path.join(self.meta_path,branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path: + pkgs.remove(name) + if pkgs: + log.error("The {0} not exist in obs_meta dir {1}".format(pkgs,branch)) error_flag = True return error_flag @@ -626,7 +634,8 @@ class CheckReleaseManagement(object): check the key in your yaml compliance with rules """ error_flag = "" - keylist = ['branch_from', 'obs_from', 'name', 'branch_to', 'obs_to', 'date', 'change_pr'] + # keylist = ['branch_from', 'obs_from', 'name', 'branch_to', 'obs_to', 'date', 'change_pr'] + keylist = ['source_dir', 'obs_from', 'name', 'destination_dir', 'obs_to', 'date'] for branch,info in change_info.items(): if info: log.info("branch:{0},pkgs:{1} key check".format(branch,info)) @@ -774,7 +783,7 @@ class CheckReleaseManagement(object): change_delete_infos = self._ensure_delete_infos(del_old_new_version_msg,del_new_version_change_msg) log.info("change pkgs yaml key check") self._check_key_in_yaml_new(change_infos) - self._check_key_in_yaml_new(change_delete_infos) + # self._check_key_in_yaml_new(change_delete_infos) log.info("change pkgs and delete yaml obs_from and obs_to check") error_flag_add = self._check_pkg_from_and_date(self.meta_path, change_infos) error_flag_del = self._check_pkg_from_and_date(self.meta_path, change_delete_infos,vtype='delete') @@ -797,7 +806,7 @@ if __name__ == "__main__": kw = {"branch":"master", "gitee_user":"dongjie110", "gitee_pwd":"Dj@6445240", - "pr_id":"35", + "pr_id":"46", "obs_meta_path":"/home/0621/obs_meta", "release_management_path":"/home/0621/release-management"} check = CheckReleaseManagement(**kw) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 2afcaee..b822930 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -119,11 +119,15 @@ class SyncPckgMgmt(object): msg.append(tmp) elif yaml_type == 'multi-new': for pckg in file_msg['packages']: - tmp = {'pkgname': pckg['name'], 'branch_from': pckg['branch_from'], - 'branch_to': pckg['branch_to'], 'obs_from': pckg['obs_from'], + tmp = {'pkgname': pckg['name'], 'branch_from': pckg['source_dir'], + 'branch_to': pckg['destination_dir'], 'obs_from': pckg['obs_from'], 'obs_to': pckg['obs_to']} prj_pkg.setdefault(pckg['obs_to'], []).append(pckg['name']) - prj_pkg['branch'] = pckg['branch_to'] + prj_pkg['branch'] = pckg['destination_dir'] + msg.append(tmp) + elif yaml_type == 'multi-new-delete': + for pckg in file_msg['packages']: + tmp = {'pkgname': pckg['name']} msg.append(tmp) else: for label in file_msg['packages']['everything']: @@ -268,6 +272,18 @@ class SyncPckgMgmt(object): log.error("copy %s service file failed!" % tmp['pkgname']) return success_pkg_name + def _move_pkg_service(self,tmp): + from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) + pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) + mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) + if not os.path.exists(pkg_path): + cmd = "mv %s %s" % (from_pkg_path,mv_to_path) + if os.system(cmd) == 0: + change_pkgs.append(pkg) + log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],temp['obs_from'],tmp['obs_to'])) + else: + log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],temp['obs_from'],tmp['obs_to'])) + def _add_master_pkg_service(self,package): """ create and add master service to repo obs_meta @@ -324,7 +340,7 @@ class SyncPckgMgmt(object): for tmp in msg: if 'multi_version' not in branch_infos: prj_meta_br = os.path.join(self.obs_meta_path, "OBS_PRJ_meta/%s" - % tmp['branch_to']) + % tmp['branch_to']) if not os.path.exists(prj_meta_br): os.makedirs(prj_meta_br) prj_meta_path = os.path.join(prj_meta_br, tmp['obs_to']) @@ -336,7 +352,10 @@ class SyncPckgMgmt(object): if "Epol" not in tmp['obs_to'] and 'master' not in branch_infos:#Testing self._write_selfbuild_meta_file(selfbuild_meta_path, tmp['obs_to']) if 'master' not in branch_infos: - pkg_name = self._add_pkg_service(tmp) + if tmp['branch_from'] == tmp['branch_to']: + pkg_name = self._move_pkg_service(tmp) + else: + pkg_name = self._add_pkg_service(tmp) if pkg_name: pkg_names.append(pkg_name) return pkg_names @@ -356,6 +375,25 @@ class SyncPckgMgmt(object): log.info("delete %s %s %s succeed!" % (tmp['branch_to'], tmp['obs_to'], tmp['pkgname'])) return tmp['pkgname'] + def _del_pkg_new(self, pkgs, branch): + """ + find in branch dirs and delete pkg + """ + need_del_path = [] + correct_del_pkgs = [] + branch_dir = os.path.join(self.obs_meta_path,branch) + for root, dirs, files in os.walk(branch_dir, True): + for name in dirs: + c_path = os.path.join(root, name) + if name in pkgs and 'Bak' not in c_path: + correct_del_pkgs.append(name) + need_del_path.append(c_path) + for pkg_path in need_del_path: + if os.path.exists(pkg_path): + shutil.rmtree(pkg_path) + log.info("delete %s succeed!" % (pkg_path)) + return correct_del_pkgs + def _verify_meta_file(self, prj_pkg): """ verify obs_meta with pckg-mgmt.yaml @@ -473,14 +511,16 @@ class SyncPckgMgmt(object): if not yaml_dict: log.info("%s file content is empty!" % name) elif isinstance(yaml_dict['packages'], list): - msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") + # msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") if 'delete' not in branch_infos: + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new") pkg_names = self._add_prj_meta_pkgs_service(msg,branch_infos) - change_pkgs = pkg_names - self._write_release_yaml(change_pkgs,branch_infos[0]) + self._write_release_yaml(pkg_names,branch_infos[0]) else: - for tmp in msg: - self._del_pkg(tmp) + msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "multi-new-delete") + pkgs = [tmp['pkgname'] for tmp in msg] + del_change_pkgs = self._del_pkg_new(pkgs,branch_infos[0]) + self._write_release_yaml(del_change_pkgs,branch_infos[0]) else: if "everything" in yaml_dict['packages'].keys(): msg, del_msg, prj_pkg = self._parse_yaml_msg(yaml_dict, "new") @@ -493,7 +533,8 @@ class SyncPckgMgmt(object): del_change_pkgs = self._verify_meta_file(prj_pkg) if master_prj_pkg: for prj_pkg in master_prj_pkg: - self._verify_meta_file(prj_pkg) + master_del_pkgs = self._verify_meta_file(prj_pkg) + self._write_release_yaml(pkg_names,branch_infos[0]) ret = self._push_code(self.obs_meta_path) self._push_code(self.release_management_path) return ret -- Gitee From 2194a8e9a2fa1ee0d666ad6dd84fced35caab4af Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 2 Jul 2022 09:39:44 +0800 Subject: [PATCH 09/13] sdd --- core/sync_pckg_mgmt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index b822930..35eb16b 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -534,7 +534,7 @@ class SyncPckgMgmt(object): if master_prj_pkg: for prj_pkg in master_prj_pkg: master_del_pkgs = self._verify_meta_file(prj_pkg) - self._write_release_yaml(pkg_names,branch_infos[0]) + self._write_release_yaml(master_del_pkgs,'master') ret = self._push_code(self.obs_meta_path) self._push_code(self.release_management_path) return ret -- Gitee From 923e74f5c015b5fe069f6f382b18554f62f97ee1 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 2 Jul 2022 09:51:34 +0800 Subject: [PATCH 10/13] sdsd --- core/sync_pckg_mgmt.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index 35eb16b..ad34727 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -292,14 +292,14 @@ class SyncPckgMgmt(object): if not os.path.exists(filepath): os.makedirs(filepath) file_msg = """<services> - <service name="tar_scm"> - <param name="scm">git</param> - <param name="url">git@gitee.com:src-openeuler/{}.git</param> - <param name="exclude">*</param> - <param name="extract">*</param> - <param name="revision">master</param> - </service> - </services>""".format(package) + <service name="tar_scm"> + <param name="scm">git</param> + <param name="url">git@gitee.com:src-openeuler/{}.git</param> + <param name="exclude">*</param> + <param name="extract">*</param> + <param name="revision">master</param> + </service> +</services>""".format(package) try: with open(os.path.join(filepath,'_service'),'w') as f: f.write(file_msg) -- Gitee From 3bf36eb2302d17d10a96bb5bf2481cea27663f65 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 2 Jul 2022 10:50:56 +0800 Subject: [PATCH 11/13] dsds --- core/check_release_management.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/check_release_management.py b/core/check_release_management.py index fda73e2..ab96c84 100644 --- a/core/check_release_management.py +++ b/core/check_release_management.py @@ -611,14 +611,14 @@ class CheckReleaseManagement(object): error_flag = True log.error(msg) log.error("Wrong Date: <date:{0}>!!!".format(msg['date'])) - msg_path = os.path.join(meta_path, msg['source_dir'],msg['obs_from'], msg['name']) - yaml_key = os.path.join(msg['source_dir'],msg['obs_from'], msg['name']) + msg_path = os.path.join(meta_path, msg['source_dir'],msg['obs_from'], msg['name']) + yaml_key = os.path.join(msg['source_dir'],msg['obs_from'], msg['name']) if not os.path.exists(msg_path): log.error("The {0} not exist in obs_meta".format(yaml_key)) error_flag = True else: pkgs = [msg['name'] for msg in change] - branch_dir = os.path.join(self.meta_path,branch) + branch_dir = os.path.join(meta_path,branch) for root, dirs, files in os.walk(branch_dir, True): for name in dirs: c_path = os.path.join(root, name) -- Gitee From 0e5cebab5f02d3c09c5a2894769be322a7d229b0 Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 2 Jul 2022 11:03:46 +0800 Subject: [PATCH 12/13] sds --- core/sync_pckg_mgmt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index ad34727..c7373ba 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -273,16 +273,18 @@ class SyncPckgMgmt(object): return success_pkg_name def _move_pkg_service(self,tmp): + success_pkg_name = '' from_pkg_path = os.path.join(self.obs_meta_path, tmp['branch_from'], tmp['obs_from'], tmp['pkgname']) pkg_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to'], tmp['pkgname']) mv_to_path = os.path.join(self.obs_meta_path, tmp['branch_to'], tmp['obs_to']) if not os.path.exists(pkg_path): cmd = "mv %s %s" % (from_pkg_path,mv_to_path) if os.system(cmd) == 0: - change_pkgs.append(pkg) + success_pkg_name = tmp['pkgname'] log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],temp['obs_from'],tmp['obs_to'])) else: log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],temp['obs_from'],tmp['obs_to'])) + return success_pkg_name def _add_master_pkg_service(self,package): """ -- Gitee From 0dc4dccdda7c302a15e740054943bd698fcc37ac Mon Sep 17 00:00:00 2001 From: dongjie110 <17621827400@163.com> Date: Sat, 2 Jul 2022 11:07:29 +0800 Subject: [PATCH 13/13] dsd --- core/sync_pckg_mgmt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/sync_pckg_mgmt.py b/core/sync_pckg_mgmt.py index c7373ba..852aa72 100644 --- a/core/sync_pckg_mgmt.py +++ b/core/sync_pckg_mgmt.py @@ -281,9 +281,9 @@ class SyncPckgMgmt(object): cmd = "mv %s %s" % (from_pkg_path,mv_to_path) if os.system(cmd) == 0: success_pkg_name = tmp['pkgname'] - log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],temp['obs_from'],tmp['obs_to'])) + log.info("move %s from %s to %s _service succeed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) else: - log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],temp['obs_from'],tmp['obs_to'])) + log.info("move %s from %s to %s _service failed!" % (tmp['pkgname'],tmp['obs_from'],tmp['obs_to'])) return success_pkg_name def _add_master_pkg_service(self,package): -- Gitee