From d9fd9241efa17189f659de7d0b8e11ab2bc633f4 Mon Sep 17 00:00:00 2001 From: miao_kaibo Date: Tue, 1 Dec 2020 16:56:13 +0800 Subject: [PATCH] add method for getting latest info of package --- core/runner.py | 21 ++++++--- core/save.py | 109 +++++++++++++++++++++++++++++++++++++++++------ openeuler_obs.py | 5 +++ 3 files changed, 118 insertions(+), 17 deletions(-) diff --git a/core/runner.py b/core/runner.py index 5c4caef..46ca6f7 100644 --- a/core/runner.py +++ b/core/runner.py @@ -55,14 +55,23 @@ class Runner(object): obs_pkgm = OBSPkgManager(**self.kwargs) obs_pkgm.obs_pkg_admc() - def _save_package_info(self): + def _save_unsync_info(self): """ - save package info for manual operation later + save unsync package info for manual operation later return: """ log.debug("save package info") - si = SaveInfo() - si.save_package_msg(self.kwargs["repository"], self.kwargs["branch"]) + si = SaveInfo(self.kwargs["gitee_user"], self.kwargs["gitee_pwd"]) + si.save_unsync_package(self.kwargs["repository"], self.kwargs["branch"]) + + def _save_latest_info(self): + """ + save package latest info + return: + """ + log.debug("save latest info") + si = SaveInfo(self.kwargs["gitee_user"], self.kwargs["gitee_pwd"]) + si.save_latest_info(self.kwargs["branch"]) def _update_package(self): """ @@ -90,13 +99,15 @@ class Runner(object): log.debug(self.update_enabled_flag) if self.kwargs["repo_rpms_update"]: self._update_obs_repo_rpms() + elif self.kwargs["latest_info"]: + self._save_latest_info() elif self.kwargs["repository"] == "obs_meta": self._obs_meta_action() elif self.kwargs["repository"] not in self.ignore_list: if not self.update_enabled_flag[self.kwargs["branch"]]: log.debug("can not update branch:%s, package: %s" % (self.kwargs["branch"], self.kwargs["repository"])) - self._save_package_info() + self._save_unsync_info() else: self._update_package() diff --git a/core/save.py b/core/save.py index 98184fa..75edae6 100644 --- a/core/save.py +++ b/core/save.py @@ -20,40 +20,125 @@ save package info import csv import os import sys +import time +import threadpool current_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(os.path.join(current_path, "..")) from common.log_obs import log from common.parser_config import ParserConfigIni +from common.common import git_repo_src + class SaveInfo(object): """ save info """ - def __init__(self): + def __init__(self, gitee_user, gitee_pwd): """ init """ parc = ParserConfigIni() self.file_name = parc.get_package_info_file() + self.branch_prj = parc.get_branch_proj() + self.obs_pkg_rpms_url = parc.get_repos_dict()["obs_pkg_rpms"] + self.obs_pkg_rpms_files_dir = None + self.gitee_user = gitee_user + self.gitee_pwd = gitee_pwd + + def _download(self): + """ + download obs_pkg_rpms + """ + tmpdir = os.popen("mktemp").read().split("\n")[0] + self.obs_pkg_rpms_files_dir = git_repo_src(self.obs_pkg_rpms_url, self.gitee_user, self.gitee_pwd, tmpdir) + + def _delete(self): + """ + delete obs_pkg_rpms + """ + cmd = "rm -rf %s" % self.obs_pkg_rpms_files_dir + os.system(cmd) - def save_package_msg(self, package_name, branch_name): + def save_unsync_package(self, package_name, branch_name): """ save info package_name: package which is not be updated branch_name: branch of package """ - file_path = os.path.join(current_path, "../log", self.file_name) - log.info("package: %s, branch: %s" % (package_name, branch_name)) - with open(file_path, 'a') as f: - csv_writer = csv.writer(f) - csv_writer.writerow([package_name, branch_name]) - cmd="git pull; git add %s; git commit -m 'update package info'; \ - git push" % file_path - os.system(cmd) + self._download() + try: + timestr = time.strftime("%Y%m%d %H-%M-%S", time.localtime()) + file_path = os.path.join(self.obs_pkg_rpms_files_dir, "unsync.csv") + log.info("package: %s, branch: %s" % (package_name, branch_name)) + with open(file_path, 'a') as f: + csv_writer = csv.writer(f) + csv_writer.writerow([timestr, package_name, branch_name]) + cmd="cd %s && git add * && git commit -m 'update package info' && git push" % self.obs_pkg_rpms_files_dir + os.system(cmd) + except AttributeError as e: + log.error(e) + finally: + self._delete() + + def _save_latest_info_by_pkg(self, datestr_root_path, prj, pkg, file_list, f_csv): + """ + save latest info of package + """ + if pkg in file_list: + cmd = "cat %s/%s" % (datestr_root_path, pkg) + log.debug(cmd) + timestr = os.popen(cmd).read().replace("\n", "") + else: + timestr = 0 + cmd = "osc list -b %s %s|grep rpm" % (prj, pkg) + log.debug(cmd) + rpms = ' '.join(list(set(os.popen(cmd).read().replace(" ", "").split("\n")) - set(['']))) + f_csv.writerow([timestr, pkg, rpms]) + + def save_latest_info(self, branch_name): + """ + save latest info of package, include update time of package code and rpms + """ + self._download() + try: + datestr_root_path = os.path.join(self.obs_pkg_rpms_files_dir, branch_name) + if not os.path.exists(datestr_root_path): + os.makedirs(datestr_root_path) + with open("%s.csv" % datestr_root_path, "w") as f: + f_csv = csv.writer(f) + file_list = os.listdir(datestr_root_path) + prj_list = self.branch_prj[branch_name].split(" ") + log.debug(prj_list) + for prj in prj_list: + cmd = "osc list %s" % prj + log.debug(cmd) + pkgs = os.popen(cmd).read().split("\n") + var_list = [] + for pkg in pkgs: + var_list.append(([datestr_root_path, prj, pkg, file_list, f_csv], None)) + try: + pool = threadpool.ThreadPool(20) + requests = threadpool.makeRequests(self._save_latest_info_by_pkg, var_list) + for req in requests: + pool.putRequest(req) + pool.wait() + except KeyboardInterrupt as e: + log.error(e) + cmd = "sort -u -r %s.csv > tmp && cat tmp > %s.csv" % (datestr_root_path, datestr_root_path) + if os.system(cmd) != 0: + log.error("sort file fail") + cmd = "cd %s && git pull && git add %s.csv && git commit -m 'update file' && git push" % \ + (self.obs_pkg_rpms_files_dir, datestr_root_path) + os.system(cmd) + except AttributeError as e: + log.error(e) + finally: + self._delete() if __name__ == "__main__": - s = SaveInfo() - s.save_package_msg("vim", "master") + s = SaveInfo("xxx", "xxx") + s.save_unsync_package("vim", "master") + s.save_latest_info("openEuler-20.03-LTS-SP1") diff --git a/openeuler_obs.py b/openeuler_obs.py index a12b607..6cacead 100644 --- a/openeuler_obs.py +++ b/openeuler_obs.py @@ -73,6 +73,9 @@ par.add_argument("-rsup", "--repo_server_pwd", default=None, par.add_argument("-rsp", "--repo_server_port", default=None, help="obs repo server port.", required=False) +par.add_argument("-latest", "--latest_info", default=False, + help="store latest package info by branch to gitee repo obs_pkg_rpms, \ + should be with -b -guser -gpwd.", required=False) args = par.parse_args() #apply @@ -99,6 +102,8 @@ kw = { "repo_server_user": args.repo_server_user, "repo_server_port": args.repo_server_port, "repo_server_pwd": args.repo_server_pwd, + + "latest_info": args.latest_info, } run = Runner(**kw) -- Gitee