diff --git a/common/parser_config.py b/common/parser_config.py index 5a37c52b3a1810cefa530d04fc3f72f14b79493c..74dbb84c4228676b5e7a92bbb4a7fec6f21c1545 100644 --- a/common/parser_config.py +++ b/common/parser_config.py @@ -70,6 +70,7 @@ class ParserConfigIni(object): self._init_obs_include_project() self._init_kernel_branch() self._init_release_maintenance_branch() + self._init_need_ccb_branch() def _init_branch_list(self): """ @@ -156,6 +157,20 @@ class ParserConfigIni(object): """ return self.release_maintenance_branchs + def _init_need_ccb_branch(self): + """ + init need ccb branch list + return: None + """ + self.need_ccb_branchs = self.config.get("need_ccb_branch", "name").split(" ") + + def get_need_ccb_branch(self): + """ + get need ccb branchs + return: need ccb branch list + """ + return self.need_ccb_branchs + def _init_package_info_file(self): """ init package info file for store package which is not be updated diff --git a/config/config.ini b/config/config.ini index 2063eb66acb3e84637bbcc815ca586925a4da1d1..01bd26ef3918aa325072ff2be5b705038d158d6b 100644 --- a/config/config.ini +++ b/config/config.ini @@ -13,6 +13,7 @@ openEuler-22.03-LTS = True openEuler-22.03-LTS-LoongArch = True openEuler-22.09 = True openEuler-22.03-LTS-SP1 = True +EBS-openEuler-22.03-LTS-SP1 = True oepkg_openstack-train_oe-20.03-LTS-SP1 = True oepkg_openstack-common_oe-20.03-LTS-SP2 = True oepkg_openstack-queens_oe-20.03-LTS-SP2 = True @@ -51,6 +52,7 @@ openEuler-22.03-LTS = openEuler:22.03:LTS openEuler:22.03:LTS:Epol openEuler:22. openEuler-22.03-LTS-LoongArch = openEuler:22.03:LTS:LoongArch openEuler-22.09 = openEuler:22.09 openEuler:22.09:Epol openEuler-22.03-LTS-SP1 = openEuler:22.03:LTS:SP1 openEuler:22.03:LTS:SP1:Epol +EBS-openEuler-22.03-LTS-SP1 = openEuler-22.03-LTS-SP1:everything openEuler-22.03-LTS-SP1:epol oepkg_openstack-train_oe-20.03-LTS-SP1 = openEuler:20.03:LTS:SP1:oepkg:openstack:train oepkg_openstack-common_oe-20.03-LTS-SP2 = openEuler:20.03:LTS:SP2:oepkg:openstack:common oepkg_openstack-queens_oe-20.03-LTS-SP2 = openEuler:20.03:LTS:SP2:oepkg:openstack:queens @@ -72,8 +74,9 @@ Multi-Version_OpenStack-Train_openEuler-22.03-LTS-SP1 = openEuler:22.03:LTS:SP1: Multi-Version_OpenStack-Wallaby_openEuler-22.03-LTS-SP1 = openEuler:22.03:LTS:SP1:Epol:Multi-Version:OpenStack:Wallaby [gitee_repository] community = https://gitee.com/openeuler/community +release_management = https://gitee.com/openeuler/release-management obs_meta = https://gitee.com/src-openeuler/obs_meta -obs_pkg_rpms = https://gitee.com/openeuler_latest_rpms/obs_pkg_rpms +obs_pkg_rpms = https://gitee.com/openeuler_latest_rpms/obs_pkg_rpms_1230 [obs_project_repos] # replace obs project's ":" by "-" openEuler-Mainline = openEuler:selfbuild:BaseOS/baseos_aarch64 openEuler:selfbuild:BaseOS/baseos_x86_64 @@ -148,4 +151,6 @@ name = openEuler:Factory openEuler:Epol openEuler:Mainline bringInRely openEuler [openEuler_kernel_branch] hck_branch = openEuler-22.09-HCK [release_maintenance_branch] -name = master multi_version openEuler-20.03-LTS-SP1 openEuler-20.03-LTS-SP3 openEuler-22.03-LTS-Next openEuler-22.03-LTS openEuler-22.09 openEuler-22.03-LTS-SP1 +name = master multi_version openEuler-20.03-LTS-SP1 openEuler-20.03-LTS-SP3 openEuler-22.03-LTS-Next openEuler-22.03-LTS openEuler-22.09 openEuler-22.03-LTS-SP1 +[need_ccb_branch] +name = EBS-openEuler-22.03-LTS-SP1 diff --git a/core/save.py b/core/save.py index 36d3b854a6e10e3ddcc0a267cc832304ce43ffbc..ea9a0633775e5544be17753a98f6f1f851f9ae12 100644 --- a/core/save.py +++ b/core/save.py @@ -21,6 +21,7 @@ import csv import os import sys import time +import yaml import threadpool current_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) @@ -41,8 +42,11 @@ class SaveInfo(object): parc = ParserConfigIni() self.file_name = parc.get_package_info_file() self.branch_prj = parc.get_branch_proj() + self.need_ccb_branch = parc.get_need_ccb_branch() self.obs_pkg_rpms_url = parc.get_repos_dict()["obs_pkg_rpms"] + self.release_management_url = parc.get_repos_dict()["release_management"] self.obs_pkg_rpms_files_dir = None + self.release_management_files_dir = None self.gitee_user = gitee_user self.gitee_pwd = gitee_pwd @@ -53,6 +57,13 @@ class SaveInfo(object): tmpdir = os.popen("mktemp").read().split("\n")[0] self.obs_pkg_rpms_files_dir = git_repo_src(self.obs_pkg_rpms_url, self.gitee_user, self.gitee_pwd, tmpdir) + def _download_release_management(self): + """ + download release-management + """ + tmpdir = os.popen("mktemp").read().split("\n")[0] + self.release_management_files_dir = git_repo_src(self.release_management_url, self.gitee_user, self.gitee_pwd, tmpdir) + def _delete(self): """ delete obs_pkg_rpms @@ -70,6 +81,8 @@ class SaveInfo(object): try: timestr = time.strftime("%Y%m%d %H-%M-%S", time.localtime()) file_path = os.path.join(self.obs_pkg_rpms_files_dir, "unsync.csv") + if branch_name in self.need_ccb_branch: + branch_name = branch_name.replace("EBS-", "") log.info("package: %s, branch: %s" % (package_name, branch_name)) with open(file_path, 'a') as f: csv_writer = csv.writer(f) @@ -83,19 +96,25 @@ class SaveInfo(object): finally: self._delete() - def _save_latest_info_by_pkg(self, datestr_root_path, prj, pkg, file_list, f_csv): + def _save_latest_info_by_pkg(self, datestr_root_path, prj, pkg, file_list, f_csv, code_update_time): """ save latest info of package """ - if pkg in file_list: - cmd = "cat %s/%s" % (datestr_root_path, pkg) + if code_update_time == 0 or code_update_time: + timestr = code_update_time + cmd = "ccb ls -p %s %s 2>/dev/null | grep \"\.rpm\"" % (prj, pkg) log.debug(cmd) - timestr = os.popen(cmd).read().replace("\n", "") + rpms = ' '.join(list(set(os.popen(cmd).read().replace(" ", "").replace('"', '').replace(",", "").split("\n")) - set(['']))) else: - timestr = 0 - cmd = "osc list -b %s %s|grep rpm" % (prj, pkg) - log.debug(cmd) - rpms = ' '.join(list(set(os.popen(cmd).read().replace(" ", "").split("\n")) - set(['']))) + if pkg in file_list: + cmd = "cat %s/%s" % (datestr_root_path, pkg) + log.debug(cmd) + timestr = os.popen(cmd).read().replace("\n", "") + else: + timestr = 0 + cmd = "osc list -b %s %s 2>/dev/null | grep rpm" % (prj, pkg) + log.debug(cmd) + rpms = ' '.join(list(set(os.popen(cmd).read().replace(" ", "").split("\n")) - set(['']))) f_csv.writerow([timestr, pkg, rpms]) def save_latest_info(self, branch_name): @@ -103,7 +122,14 @@ class SaveInfo(object): save latest info of package, include update time of package code and rpms """ self._download() + need_ccb = None + code_update_time = 0 try: + prj_list = self.branch_prj[branch_name].split(" ") + log.debug(prj_list) + if branch_name in self.need_ccb_branch: + branch_name = branch_name.replace("EBS-", "") + need_ccb = "yes" datestr_root_path = os.path.join(self.obs_pkg_rpms_files_dir, branch_name) if not os.path.exists(datestr_root_path): os.makedirs(datestr_root_path) @@ -111,15 +137,49 @@ class SaveInfo(object): with open(latest_rpm_file, "w") as f: f_csv = csv.writer(f) file_list = os.listdir(datestr_root_path) - prj_list = self.branch_prj[branch_name].split(" ") - log.debug(prj_list) var_list = [] for prj in prj_list: - cmd = "osc list %s" % prj - log.debug(cmd) - pkgs = os.popen(cmd).read().split("\n") + if need_ccb: + cmd = "ccb select builds build_type=full,incremental status=201,202 -f create_time os_project=%s build_target.architecture=x86_64 --sort create_time:desc --size 1 2>/dev/null | grep create_time | awk -F': ' '{print $NF}' | awk -F'.' '{print $1}'" % prj + log.debug(cmd) + res = os.popen(cmd).read() + if res: + code_update_time = res.replace('"', '').replace("T", " ").replace(":", "-").replace("\n", "") + log.info("code_update_time:%s" % code_update_time) + cmd = "ccb select projects os_project=%s submit_order -f my_specs 2>/dev/null | grep spec_url" % prj + log.debug(cmd) + pkgs = [] + res = os.popen(cmd).read().split("\n") + spec_url = [x for x in res if x != ''] + if spec_url: + for url in spec_url: + p = url.split("/")[-1].split(".git")[0] + if p not in pkgs: + pkgs.append(p) + else: + if not self.release_management_files_dir: + self._download_release_management() + if "epol" in prj.lower(): + standard_dirs = ['epol'] + else: + standard_dirs = ['everything-exclude-baseos', 'baseos'] + release_management_path = os.path.join(self.release_management_files_dir, branch_name) + for c_dir in standard_dirs: + yaml_path = os.path.join(release_management_path, c_dir, 'pckg-mgmt.yaml') + if os.path.exists(yaml_path): + with open(yaml_path, 'r', encoding='utf-8') as f: + result = yaml.safe_load(f) + for line in result['packages']: + name = line.get("name") + if name not in pkgs: + pkgs.append(name) + else: + cmd = "osc list %s" % prj + log.debug(cmd) + pkgs = os.popen(cmd).read().split("\n") + code_update_time = None for pkg in pkgs: - var_list.append(([datestr_root_path, prj, pkg, file_list, f_csv], None)) + var_list.append(([datestr_root_path, prj, pkg, file_list, f_csv, code_update_time], None)) try: pool = threadpool.ThreadPool(20) requests = threadpool.makeRequests(self._save_latest_info_by_pkg, var_list)