diff --git a/config/config.ini b/config/config.ini index 5652d8f73e359f2abfe02470f471647f79ad1a8a..444b59fd40a41aab467c03418510e663a3e1871c 100644 --- a/config/config.ini +++ b/config/config.ini @@ -59,16 +59,13 @@ openEuler-22.03-LTS-Next = openEuler:22.03:LTS:Next openEuler:22.03:LTS:Next:Epo openEuler-22.03-LTS = openEuler:22.03:LTS openEuler:22.03:LTS:Epol openEuler:22.03:LTS:Extras openEuler-22.03-LTS-LoongArch = openEuler:22.03:LTS:LoongArch openEuler:22.03:LTS:LoongArch:Epol openEuler-22.03-LTS-ppc64le = openEuler:22.03:LTS:ppc64le -openEuler-22.03-LTS-SP1 = openEuler:22.03:LTS:SP1 openEuler:22.03:LTS:SP1:Epol openEuler-22.03-LTS-SP2 = openEuler:22.03:LTS:SP2 openEuler:22.03:LTS:SP2:Epol openEuler-23.09 = openEuler:23.09 openEuler:23.09:Epol -openEuler-20.03-LTS-SP4 = openEuler:20.03:LTS:SP4 openEuler:20.03:LTS:SP4:Epol -openEuler-22.03-LTS-SP3 = openEuler:22.03:LTS:SP3 openEuler:22.03:LTS:SP3:Epol -EBS-openEuler-22.03-LTS-SP1 = openEuler-22.03-LTS-SP1:everything openEuler-22.03-LTS-SP1:epol -EBS-openEuler-22.03-LTS-SP3 = openEuler-22.03-LTS-SP3:everything openEuler-22.03-LTS-SP3:epol -EBS-openEuler-22.03-LTS-SP4 = openEuler-22.03-LTS-SP4:everything openEuler-22.03-LTS-SP4:epol -EBS-openEuler-20.03-LTS-SP4 = openEuler-20.03-LTS-SP4:everything openEuler-20.03-LTS-SP4:epol -EBS-openEuler-24.03-LTS = openEuler-24.03-LTS:everything openEuler-24.03-LTS:epol +openEuler-22.03-LTS-SP1 = openEuler-22.03-LTS-SP1:everything openEuler-22.03-LTS-SP1:epol +openEuler-22.03-LTS-SP3 = openEuler-22.03-LTS-SP3:everything openEuler-22.03-LTS-SP3:epol +openEuler-22.03-LTS-SP4 = openEuler-22.03-LTS-SP4:everything openEuler-22.03-LTS-SP4:epol +openEuler-20.03-LTS-SP4 = openEuler-20.03-LTS-SP4:everything openEuler-20.03-LTS-SP4:epol +openEuler-24.03-LTS = openEuler-24.03-LTS:everything openEuler-24.03-LTS:epol Multi-Version_OpenStack-Train_openEuler-22.03-LTS-Next = openEuler:22.03:LTS:Next:Epol:Multi-Version:OpenStack:Train Multi-Version_OpenStack-Wallaby_openEuler-22.03-LTS-Next = openEuler:22.03:LTS:Next:Epol:Multi-Version:OpenStack:Wallaby Multi-Version_OpenStack-Train_openEuler-22.03-LTS = openEuler:22.03:LTS:Epol:Multi-Version:OpenStack:Train diff --git a/core/runner.py b/core/runner.py index ab9bb0286032ed23168db680be23f20ea5edf2c1..d81cd5ade581291d9e89777dfe89b9b13bf92972 100644 --- a/core/runner.py +++ b/core/runner.py @@ -78,7 +78,7 @@ class Runner(object): return: """ log.debug("save latest info") - si = SaveInfo(self.kwargs["gitee_user"], self.kwargs["gitee_pwd"]) + si = SaveInfo(self.kwargs["gitee_user"], self.kwargs["gitee_pwd"], self.kwargs["build_type"]) si.save_latest_info(self.kwargs["branch"]) def _update_package(self): diff --git a/core/save.py b/core/save.py index 7181b6fa59c09ba13f43de339775a0b7d1a7a8e3..575c3e081d33fe478749aac6e499a5274782e821 100644 --- a/core/save.py +++ b/core/save.py @@ -22,6 +22,7 @@ import os import sys import time import yaml +import json import threadpool current_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) @@ -35,20 +36,20 @@ class SaveInfo(object): """ save info """ - def __init__(self, gitee_user, gitee_pwd): + def __init__(self, gitee_user, gitee_pwd, build_type): """ init """ parc = ParserConfigIni() self.file_name = parc.get_package_info_file() self.branch_prj = parc.get_branch_proj() - self.need_ccb_branch = parc.get_need_ccb_branch() self.obs_pkg_rpms_url = parc.get_repos_dict()["obs_pkg_rpms"] self.release_management_url = parc.get_repos_dict()["release_management"] self.obs_pkg_rpms_files_dir = None self.release_management_files_dir = None self.gitee_user = gitee_user self.gitee_pwd = gitee_pwd + self.build_type = build_type def _download(self): """ @@ -81,8 +82,6 @@ class SaveInfo(object): try: timestr = time.strftime("%Y%m%d %H-%M-%S", time.localtime()) file_path = os.path.join(self.obs_pkg_rpms_files_dir, "unsync.csv") - if branch_name in self.need_ccb_branch: - branch_name = branch_name.replace("EBS-", "") log.info("package: %s, branch: %s" % (package_name, branch_name)) with open(file_path, 'a') as f: csv_writer = csv.writer(f) @@ -96,116 +95,135 @@ class SaveInfo(object): finally: self._delete() - def _save_latest_info_by_pkg(self, datestr_root_path, prj, pkg, file_list, f_csv, code_update_time): + def _get_pkg_rpmlist(self, prj, pkg): """ - save latest info of package + get pkg rpms """ - if code_update_time == 0 or code_update_time: - timestr = code_update_time - if pkg == "kernel": - tmp_pkg = pkg + ":" + pkg - else: - tmp_pkg = pkg - cmd = "ccb ls -p %s %s 2>/dev/null | grep \"\.rpm\"" % (prj, tmp_pkg) + tmp_pkg = pkg + if pkg == "kernel": + tmp_pkg = pkg + ":" + pkg + cmd = f"ccb ls -p {prj} {tmp_pkg} 2>/dev/null | grep \"\.rpm\"" + log.debug(cmd) + res=os.popen(cmd).read() + rpms = "" + if res: + rpmlist = sorted(list(set(res.replace(" ", "").replace('"', '').replace(",", "").split("\n")) - set(['']))) + rpms = ' '.join(rpmlist) + return rpms + + def _save_latest_info_specified(self, prj_list, latest_rpm_file): + """ + save latest info of specified + """ + for prj in prj_list: + cmd = f"ccb select builds status=201,202 -f _id,create_time,build_type os_project={prj} build_target.architecture=x86_64 --sort create_time:desc --size 1 2>/dev/null" log.debug(cmd) - rpms = ' '.join(list(set(os.popen(cmd).read().replace(" ", "").replace('"', '').replace(",", "").split("\n")) - set(['']))) - else: - if pkg in file_list: - cmd = "cat %s/%s" % (datestr_root_path, pkg) - log.debug(cmd) - timestr = os.popen(cmd).read().replace("\n", "") - else: - timestr = 0 - if prj == "openEuler:22.03:LTS": - archs = ['aarch64','x86_64'] - tmp_rpms = [] - for ar in archs: - cmd = "osc list -b %s %s -r standard_%s -a %s 2>/dev/null | grep rpm" % (prj, pkg, ar, ar) + res = os.popen(cmd).read() + if res: + data = json.loads(res) + bt = data[0]['_source']['build_type'] + if bt == "specified": + log.info("build_type:%s" % bt) + code_update_time = data[0]['_source']['create_time'].partition('.')[0].replace("T", " ").replace("-", "").replace(":", "-") + log.info("code_update_time:%s" % code_update_time) + build_id = data[0]['_id'] + log.info("build_id:%s" % build_id) + + cmd = f"ccb select builds _id={build_id} -f select_pkgs" log.debug(cmd) - tmp_rpms.extend(os.popen(cmd).read().replace(" ", "").split("\n")) - rpms = ' '.join(list(set(tmp_rpms))) - else: - cmd = "osc list -b %s %s 2>/dev/null | grep rpm" % (prj, pkg) - log.debug(cmd) - rpms = ' '.join(list(set(os.popen(cmd).read().replace(" ", "").split("\n")) - set(['']))) - f_csv.writerow([timestr, pkg, rpms]) + res = os.popen(cmd).read() + if res: + result = json.loads(res) + select_pkgs = result[0]['_source']['select_pkgs'] + log.info("select_pkgs:%s" % select_pkgs) + for pkg in select_pkgs: + rpms = self._get_pkg_rpmlist(prj, pkg) + str_info = f"{code_update_time},{pkg},${rpms}" + cmd = "sed -i '/,%s,/d' %s" % (pkg, latest_rpm_file) + ret = os.system(cmd) + log.info("ret: %s" % ret) + cmd = "sed -i '$a\%s\r\n' %s" % (str_info, latest_rpm_file) + ret = os.system(cmd) + log.info("ret: %s" % ret) + else: + log.info("The last build of the %s project is not specified, but %s" % (prj, bt)) + + def _save_latest_info_by_pkg(self, prj, pkg, f_csv, code_update_time): + """ + save latest info of package + """ + if code_update_time == 0 or code_update_time: + rpms = self._get_pkg_rpmlist(prj, pkg) + f_csv.writerow([code_update_time, pkg, rpms]) def save_latest_info(self, branch_name): """ save latest info of package, include update time of package code and rpms """ self._download() - need_ccb = None code_update_time = 0 try: prj_list = self.branch_prj[branch_name].split(" ") log.debug(prj_list) - if branch_name in self.need_ccb_branch: - branch_name = branch_name.replace("EBS-", "") - need_ccb = "yes" - datestr_root_path = os.path.join(self.obs_pkg_rpms_files_dir, branch_name) - if not os.path.exists(datestr_root_path): - os.makedirs(datestr_root_path) latest_rpm_file = os.path.join(self.obs_pkg_rpms_files_dir, "latest_rpm", "%s.csv" % branch_name) - with open(latest_rpm_file, "w") as f: - f_csv = csv.writer(f) - file_list = os.listdir(datestr_root_path) - var_list = [] - for prj in prj_list: - if need_ccb: - cmd = "ccb select builds build_type=full,incremental status=201,202 -f create_time os_project=%s build_target.architecture=x86_64 --sort create_time:desc --size 1 2>/dev/null | grep create_time | awk -F': ' '{print $NF}' | awk -F'.' '{print $1}'" % prj + if self.build_type == "specified": + self._save_latest_info_specified(prj_list, latest_rpm_file) + else: + with open(latest_rpm_file, "w") as f: + f_csv = csv.writer(f) + for prj in prj_list: + var_list = [] + pkg_list = [] + cmd = f"ccb select builds build_type=full,incremental status=201,202 -f create_time,build_type os_project={prj} build_target.architecture=x86_64 --sort create_time:desc --size 1 2>/dev/null" log.debug(cmd) res = os.popen(cmd).read() if res: - code_update_time = res.replace('"', '').replace("T", " ").replace("-", "").replace(":", "-").replace("\n", "") - log.info("code_update_time:%s" % code_update_time) - cmd = "ccb select projects os_project=%s submit_order -f my_specs 2>/dev/null | grep spec_url" % prj - log.debug(cmd) - pkgs = [] - res = os.popen(cmd).read().split("\n") - spec_url = [x for x in res if x != ''] - if spec_url: - for url in spec_url: - p = url.split("/")[-1].split(".git")[0] - if p not in pkgs: - pkgs.append(p) - else: - if not self.release_management_files_dir: - self._download_release_management() - if "epol" in prj.lower(): - standard_dirs = ['epol'] + data = json.loads(res) + code_update_time = data[0]['_source']['create_time'].partition('.')[0].replace("T", " ").replace("-", "").replace(":", "-") + bt = data[0]['_source']['build_type'] + log.info("code_update_time:%s" % code_update_time) + log.info("build_type:%s" % bt) + + cmd = f"ccb select projects os_project={prj} submit_order -f my_specs 2>/dev/null | grep spec_url" + log.debug(cmd) + res = os.popen(cmd).read().split("\n") + spec_url = [x for x in res if x != ''] + if spec_url: + for url in spec_url: + p = url.split("/")[-1].split(".git")[0] + if p not in pkg_list: + pkg_list.append(p) else: - standard_dirs = ['everything-exclude-baseos', 'baseos'] - release_management_path = os.path.join(self.release_management_files_dir, branch_name) - for c_dir in standard_dirs: - yaml_path = os.path.join(release_management_path, c_dir, 'pckg-mgmt.yaml') - if os.path.exists(yaml_path): - with open(yaml_path, 'r', encoding='utf-8') as f: - result = yaml.safe_load(f) - for line in result['packages']: - name = line.get("name") - if name not in pkgs: - pkgs.append(name) - else: - cmd = "osc list %s" % prj - log.debug(cmd) - pkgs = os.popen(cmd).read().split("\n") - code_update_time = None - for pkg in pkgs: - var_list.append(([datestr_root_path, prj, pkg, file_list, f_csv, code_update_time], None)) - try: - pool = threadpool.ThreadPool(20) - requests = threadpool.makeRequests(self._save_latest_info_by_pkg, var_list) - for req in requests: - pool.putRequest(req) - pool.wait() - except KeyboardInterrupt as e: - log.error(e) - cmd = "sort -u -r %s > tmp && cat tmp > %s" % (latest_rpm_file, latest_rpm_file) + if not self.release_management_files_dir: + self._download_release_management() + if "epol" in prj.lower(): + standard_dirs = ['epol'] + else: + standard_dirs = ['everything-exclude-baseos', 'baseos'] + release_management_path = os.path.join(self.release_management_files_dir, branch_name) + for c_dir in standard_dirs: + yaml_path = os.path.join(release_management_path, c_dir, 'pckg-mgmt.yaml') + if os.path.exists(yaml_path): + with open(yaml_path, 'r', encoding='utf-8') as f: + result = yaml.safe_load(f) + for line in result['packages']: + name = line.get("name") + if name not in pkg_list: + pkg_list.append(name) + for pkg in pkg_list: + var_list.append(([prj, pkg, f_csv, code_update_time], None)) + try: + pool = threadpool.ThreadPool(20) + requests = threadpool.makeRequests(self._save_latest_info_by_pkg, var_list) + for req in requests: + pool.putRequest(req) + pool.wait() + except KeyboardInterrupt as e: + log.error(e) + cmd = f"sort -u -r {latest_rpm_file} -o {latest_rpm_file}" if os.system(cmd) != 0: log.error("sort file fail") - cmd = "cd %s && git pull && git add * && git commit -m 'update file' && git push" % \ - self.obs_pkg_rpms_files_dir + cmd = f"cd {self.obs_pkg_rpms_files_dir} && git pull && git add * && git commit -m 'update file' && git push" log.debug(cmd) for i in range(5): if os.system(cmd) == 0: @@ -220,4 +238,3 @@ if __name__ == "__main__": s = SaveInfo("xxx", "xxx") s.save_unsync_package("vim", "master") s.save_latest_info("openEuler-20.03-LTS-SP1") - diff --git a/openeuler_obs.py b/openeuler_obs.py index da38a85cd856f44be5d77d0416911e8f054d679f..86eab058e3fe7dddcb07ad493910ef64b246d780 100644 --- a/openeuler_obs.py +++ b/openeuler_obs.py @@ -120,6 +120,7 @@ par.add_argument("-amsb", "--align_meta_service_branch", default=None, par.add_argument("-ju", "--jenkins_user", help="jekins user name", required=False) par.add_argument("-jt", "--jenkins_api_token", help="jenkins api token", required=False) par.add_argument("-jbu", "--jenkins_build_url", help="jenkins build url", required=False) +par.add_argument("-bt", "--build_type", default=None, help="eulermaker build type", required=False) args = par.parse_args() #apply @@ -170,7 +171,8 @@ kw = { "align_meta_service_branch": args.align_meta_service_branch, "jenkins_user": args.jenkins_user, "jenkins_api_token": args.jenkins_api_token, - "jenkins_build_url": args.jenkins_build_url + "jenkins_build_url": args.jenkins_build_url, + "build_type": args.build_type } run = Runner(**kw)