diff --git a/common/parser_config.py b/common/parser_config.py index 9623f4bfca64a7daa28205055b4072012a2622ac..b4fcd4d67a4cb48452d5904ea70f98c8ae0a3c6e 100644 --- a/common/parser_config.py +++ b/common/parser_config.py @@ -66,6 +66,8 @@ class ParserConfigIni(object): self._init_gitee_repository() self._init_obs_repository() self._init_obs_prj_root_path() + self._init_obs_ignored_package() + self._init_obs_include_project() def _init_branch_list(self): """ @@ -110,6 +112,34 @@ class ParserConfigIni(object): """ return self.ignored_repos + def _init_obs_ignored_package(self): + """ + init ignored package list + return: None + """ + self.obs_ignored_packages = self.config.get("obs_ignore_package", "name").split(" ") + + def get_obs_ignored_package(self): + """ + get ignored package + return: ignored package list + """ + return self.obs_ignored_packages + + def _init_obs_include_project(self): + """ + init include project list + return: None + """ + self.obs_include_projects = self.config.get("obs_include_project", "name").split(" ") + + def get_obs_include_project(self): + """ + get include project + return: include project list + """ + return self.obs_include_projects + def _init_package_info_file(self): """ init package info file for store package which is not be updated diff --git a/config/config.ini b/config/config.ini index 21338942c9c619380430ec25a47b8a82cfb82a15..c4b8945d5813e294703da1edfb0ec71237e80c00 100644 --- a/config/config.ini +++ b/config/config.ini @@ -102,5 +102,7 @@ openEuler-22.03-LTS-Epol-Multi-Version-OpenStack-Wallaby = openEuler:22.03:LTS:s [obs_project_root_path] # all obs projects path and where store all packages of all obs projects path = /srv/obs/build - - +[obs_ignore_package] +name = kernel kata-containers runc openEuler-release openEuler-logos openEuler-indexhtml mozjs78 lxcfs-tools kata_integration isula-build docker containerd +[obs_include_project] +name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely \ No newline at end of file diff --git a/core/check_meta_service.py b/core/check_meta_service.py index 9e2fc62ff1cdafa1e3c08ddf317dcd5ff4b80829..07c7905a7b5e56a896719fb4772da9b7a9360144 100644 --- a/core/check_meta_service.py +++ b/core/check_meta_service.py @@ -25,6 +25,7 @@ from common.log_obs import log from xml.etree import ElementTree as ET from bs4 import BeautifulSoup import xml.dom.minidom +from common.parser_config import ParserConfigIni class CheckMetaPull(object): """ @@ -46,6 +47,8 @@ class CheckMetaPull(object): self.current_path = os.getcwd() self.check_error = [] self.meta_path = self.kwargs['obs_meta_path'] + par = ParserConfigIni() + self.obs_ignored_package = par.get_obs_ignored_package() def _clean(self): """ @@ -166,6 +169,27 @@ class CheckMetaPull(object): log.info("ALL_URL:%s" % url_list) return url_list + def _get_url_info_new(self, pkg_path): + """ + get pkg_name and pkg_branch from service file for new service content + """ + url_list = [] + revision_list = [] + path_info = self._get_path_info(pkg_path) + with open(pkg_path, "r") as f: + log.info('\n' + f.read()) + dom = xml.dom.minidom.parse(pkg_path) + root = dom.documentElement + param_lines = root.getElementsByTagName('param') + for url in param_lines: + if url.getAttribute("name") == "url": + url_list.append(url.firstChild.data.strip('/')) + elif url.getAttribute("name") == "revision": + revision_list.append(url.firstChild.data.strip('/')) + log.info("ALL_URL:%s" % url_list) + log.info("ALL_REVISION:%s" % revision_list) + return url_list,revision_list + def _get_path_info(self, pkg_path): """ get pkg_name and pkg_branch from change path @@ -192,6 +216,22 @@ class CheckMetaPull(object): error_flag = "yes" return error_flag + def _check_correspond_from_service_new(self, **info): + """ + Distingguish check pkg_name between different branch for new service content + """ + error_flag = "" + if info['pkg_url'] == "master" and len(list(set(info['ser_branch']))) == 1: + error_flag = self._pkgname_check(info['pkg_name'], info['ser_name']) + elif list(set(info['ser_branch']))[0] == info['pkg_url'] and len(list(set(info['ser_branch']))) == 1: + error_flag = self._pkgname_check(info['pkg_name'], info['ser_name']) + else: + log.error("**************_Service URL ERROR*****************") + log.error("FAILED:Please check the url in your %s _service again" + % info['pkg_name']) + error_flag = "yes" + return error_flag + def _check_correspond_from_service(self, **info): """ Distingguish check pkg_name between different branch @@ -214,33 +254,65 @@ class CheckMetaPull(object): error_flag = "yes" return error_flag + def parse_all_info(self,pkg_path): + service_name = [] + service_branch = [] + service_next = [] + url_list = self._get_url_info(pkg_path) + for all_url in url_list: + service_next.append(all_url.split('/')[0]) + service_name.append(all_url.split('/')[-1]) + service_branch.append(all_url.split('/')[-2]) + path_info = self._get_path_info(pkg_path) + log.info("Url_Next:%s" % service_next) + log.info("Pkgname_in_Service:%s" % service_name) + log.info("pkgbranch_in_Service:%s" % service_branch) + log.info("Pkgname_in_Obe_meta_path:%s" % path_info[0]) + log.info("Pkgbranch_in_Obs_meta_path:%s" % path_info[1]) + all_info = {'ser_branch':service_branch, 'pkg_url':path_info[1], + 'ser_next':service_next, 'pkg_name':path_info[0], + 'ser_name':service_name} + return all_info + + def parse_all_info_new(self,pkg_path): + service_name = [] + service_branch = [] + url_list,revision_list = self._get_url_info_new(pkg_path) + for all_url in url_list: + head,sep,tail = all_url.split('/')[1].partition('.') + service_name.append(head) + for all_revision in revision_list: + service_branch.append(all_revision) + path_info = self._get_path_info(pkg_path) + log.info("Pkgname_in_Service:%s" % service_name) + log.info("pkgbranch_in_Service:%s" % service_branch) + log.info("Pkgname_in_Obs_meta_path:%s" % path_info[0]) + log.info("Pkgbranch_in_Obs_meta_path:%s" % path_info[1]) + all_info = {'ser_branch':service_branch, 'pkg_url':path_info[1], 'pkg_name':path_info[0], 'ser_name':service_name} + return all_info + def _check_pkg_services(self, pkg_path): """ Check the format of the service file and the correctness of its URL """ - service_name = [] - service_branch = [] - service_next = [] error_flag2 = "" error_flag3 = "" os.chdir("%s/obs_meta" % self.current_path) error_flag1 = self._check_file_format(pkg_path) if error_flag1 != "yes": - url_list = self._get_url_info(pkg_path) - for all_url in url_list: - service_next.append(all_url.split('/')[0]) - service_name.append(all_url.split('/')[-1]) - service_branch.append(all_url.split('/')[-2]) path_info = self._get_path_info(pkg_path) - log.info("Url_Next:%s" % service_next) - log.info("Pkgname_in_Service:%s" % service_name) - log.info("pkgbranch_in_Service:%s" % service_branch) - log.info("Pkgname_in_Obe_meta_path:%s" % path_info[0]) - log.info("Pkgbranch_in_Obs_meta_path:%s" % path_info[1]) - all_info = {'ser_branch':service_branch, 'pkg_url':path_info[1], - 'ser_next':service_next, 'pkg_name':path_info[0], - 'ser_name':service_name} - error_flag2 = self._check_correspond_from_service(**all_info) + meta_pkg_name = path_info[0] + meta_pkg_branch = path_info[1] + if meta_pkg_branch != 'master': + all_info = self.parse_all_info(pkg_path) + error_flag2 = self._check_correspond_from_service(**all_info) + else: + if meta_pkg_name in self.obs_ignored_package: + all_info = self.parse_all_info(pkg_path) + error_flag2 = self._check_correspond_from_service(**all_info) + else: + all_info = self.parse_all_info_new(pkg_path) + error_flag2 = self._check_correspond_from_service_new(**all_info) if self.token: error_flag3 = self._detect_protect_branch(pkg_path) error_flag4 = self._one_pkg_one_branch(pkg_path) @@ -539,7 +611,7 @@ class CheckMetaPull(object): """ check openeuler_meta pull request """ - log.info("************************************** CHECK PR RULE**********************") + log.info("******CHECK PR RULE*********") failed_flag = [] failed_msg = [] if release_manage: @@ -556,8 +628,8 @@ class CheckMetaPull(object): log.error("you can not pull request in branch:{},Please refer to this issue:https://gitee.com/openeuler/release-management/issues/I4U2VN?from=project-issue".format(failed_msg)) raise SystemExit("*******PLEASE CHECK YOUR PR*******") else: - log.error("get release management data failed,please check network and toke n**********************") - log.info("************************************** CHECK PR RULE**********************") + log.error("get release management data failed,please check network and token") + log.info("******CHECK PR RULE********") log.info("check pr rule finished,please wait other check!!!") diff --git a/core/gitee_to_obs.py b/core/gitee_to_obs.py index f3783b08038fd0b761542c18b93665e757960d3a..e9bcc8448d68ed9e1fe1bb2ed9840ac521753aaf 100644 --- a/core/gitee_to_obs.py +++ b/core/gitee_to_obs.py @@ -68,6 +68,8 @@ class SYNCCode(object): self.obs_pkg_rpms_url = par.get_repos_dict()["obs_pkg_rpms"] self.obs_pkg_prms_files_dir = None self.sync_failed_rpms = [] + self.obs_ignored_package = par.get_obs_ignored_package() + self.obs_include_project = par.get_obs_include_project() def _write_date_to_file(self): """ @@ -233,14 +235,33 @@ class SYNCCode(object): else: obs_project = project log.info("The %s in %s" % (self.repository, obs_project)) - pull_result = self._get_latest_gitee_pull() - remoterun_result = self._gitee_pr_to_obs(obs_project) - if obs_project and pull_result and remoterun_result: - log.info("SYNC %s in %s SUCCESS" % (self.repository, obs_project)) - return True + if obs_project not in self.obs_include_project: + pull_result = self._get_latest_gitee_pull() + remoterun_result = self._gitee_pr_to_obs(obs_project) + if obs_project and pull_result and remoterun_result: + log.info("SYNC %s in %s SUCCESS" % (self.repository, obs_project)) + return True + else: + log.error("SYNC %s in %s ERROR: Please check the log.error" % (self.repository, obs_project)) + return False else: - log.error("SYNC %s in %s ERROR: Please check the log.error" % (self.repository, obs_project)) - return False + if self.repository in self.obs_ignored_package: + pull_result = self._get_latest_gitee_pull() + remoterun_result = self._gitee_pr_to_obs(obs_project) + if obs_project and pull_result and remoterun_result: + log.info("SYNC %s in %s SUCCESS" % (self.repository, obs_project)) + return True + else: + log.error("SYNC %s in %s ERROR: Please check the log.error" % (self.repository, obs_project)) + return False + else: + remoterun_result = self._gitee_pr_to_obs(obs_project) + if obs_project and remoterun_result: + log.info("SYNC MASTER %s in %s SUCCESS" % (self.repository, obs_project)) + return True + else: + log.error("SYNC %s in %s ERROR: Please check the log.error" % (self.repository, obs_project)) + return False def sync_code_to_obs(self): """